modeling_autoencoder.py

from torch import nn

from transformers import PretrainedConfig, PreTrainedModel, AutoConfig, AutoModel

# from huggingface_hub import notebook_login

# notebook_login()

# AutoEncoderConfig.register_for_auto_class()
# AutoEncoder.register_for_auto_class("AutoModel")

# AutoConfig.register("autoencoder", AutoEncoderConfig)
# AutoModel.register(AutoEncoderConfig, AutoModel)

# autoencoder.push_to_hub("autoencoder")
class AutoEncoderConfig(PretrainedConfig):
    model_type = "autoencoder"

    def __init__(
        self, 
        input_dim=None, 
        latent_dim=None, 
        layer_types=None, 
        dropout_rate=None, 
        num_layers=None, 
        compression_rate=None, 
        bidirectional=False,
        **kwargs
    ):
        super().__init__(**kwargs)
        self.input_dim = input_dim
        self.latent_dim = latent_dim
        self.layer_types = layer_types
        self.dropout_rate = dropout_rate
        self.num_layers = num_layers
        self.compression_rate = compression_rate
        self.bidirectional = bidirectional

def create_layers(model_section, layer_types, input_dim, latent_dim, num_layers, dropout_rate, compression_rate, bidirectional):

    layers = []
    current_dim = input_dim

    input_diamensions = []
    output_diamensions = []

    for _ in range(num_layers):
        input_diamensions.append(current_dim)
        next_dim = max(int(current_dim * compression_rate), latent_dim)
        current_dim = next_dim
        output_diamensions.append(current_dim)

    output_diamensions[num_layers - 1] = latent_dim

    if model_section == "decoder":
        input_diamensions, output_diamensions = output_diamensions, input_diamensions
        input_diamensions.reverse()
        output_diamensions.reverse()

        if bidirectional & (layer_types in ['lstm', 'rnn', 'gru']):
            output_diamensions = [2*value for value in output_diamensions]

    for idx, (input_dim, output_dim) in enumerate(zip(input_diamensions, output_diamensions)):
        if layer_types == 'linear':
            layers.append(nn.Linear(input_dim, output_dim))
        elif layer_types == 'lstm':
            layers.append(nn.LSTM(input_dim, output_dim // (2 if bidirectional else 1), batch_first=True, bidirectional=bidirectional))
        elif layer_types == 'rnn':
            layers.append(nn.RNN(input_dim, output_dim // (2 if bidirectional else 1), batch_first=True, bidirectional=bidirectional))
        elif layer_types == 'gru':
            layers.append(nn.GRU(input_dim, output_dim // (2 if bidirectional else 1), batch_first=True, bidirectional=bidirectional))
        if (idx != num_layers - 1) & (dropout_rate != None):
            layers.append(nn.Dropout(dropout_rate))
    return nn.Sequential(*layers)

class AutoEncoder(PreTrainedModel):
    config_class = AutoEncoderConfig
    
    def __init__(self, config):
        super(AutoEncoder, self).__init__(config)
        
        self.encoder = create_layers("encoder",
            config.layer_types, config.input_dim, config.latent_dim, 
            config.num_layers, config.dropout_rate, config.compression_rate,
            config.bidirectional,
        )
        # Assuming symmetry between encoder and decoder
        self.decoder = create_layers("decoder",
            config.layer_types, config.input_dim, config.latent_dim, 
            config.num_layers, config.dropout_rate, config.compression_rate,
            config.bidirectional,
        )

    def forward(self, x):
        if self.config.layer_types in ['lstm', 'rnn', 'gru']:
            for layer in self.encoder:
                print(layer)
                if isinstance(layer, nn.LSTM):
                    x, (h_n, c_n)= layer(x)

                elif isinstance(layer, nn.RNN):
                    x, h_o = layer(x)
                elif isinstance(layer, nn.GRU):
                    x, h_o = layer(x)
                else:
                    x = layer(x)
                    
            for layer in self.decoder:
                if isinstance(layer, nn.LSTM):
                    x, (h_n, c_n) = layer(x)
                elif isinstance(layer, nn.RNN):
                    x, h_o = layer(x)
                elif isinstance(layer, nn.GRU):
                    x, h_o = layer(x)
                else:
                    x = layer(x)
                    
        else:
            x = self.encoder(x)
            x = self.decoder(x)
        return x