Create app.py

app.py

@@ -0,0 +1,264 @@
+!pip install torch==2.0.1 transformers==4.27.1 datasets==2.4.0 wget==3.2 huggingface-hub==0.14.1 beautifulsoup4==4.11.1 requests==2.28.1 matplotlib tqdm python-dotenv diffusers
+
+import os
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Dataset
+from torch.optim import AdamW
+import matplotlib.pyplot as plt
+import matplotlib.animation as animation
+import time
+import threading
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModel, TrainingArguments, pipeline
+from diffusers import DiffusionPipeline
+from huggingface_hub import login, HfApi, Repository
+from dotenv import load_dotenv
+
+# Cargar variables de entorno
+load_dotenv()
+
+class UnifiedModel(nn.Module):
+    def __init__(self, models):
+        super(UnifiedModel, self).__init__()
+        self.models = nn.ModuleList(models)
+        self.classifier = nn.Linear(sum([model.config.hidden_size for model in models if hasattr(model, 'config')]), 2)
+
+    def forward(self, inputs):
+        hidden_states = []
+        for model in self.models:
+            if isinstance(model, nn.Module):
+                outputs = model(inputs)
+                hidden_states.append(outputs.last_hidden_state[:, 0, :])
+            elif isinstance(model, DiffusionPipeline) or isinstance(model, pipeline):
+                outputs = model(inputs)
+                hidden_states.append(torch.tensor(outputs))
+        concatenated_hidden_states = torch.cat(hidden_states, dim=-1)
+        logits = self.classifier(concatenated_hidden_states)
+        return logits
+
+
+class SyntheticDataset(Dataset):
+    def __init__(self, tokenizers, size=100):
+        self.tokenizers = tokenizers
+        self.size = size
+        self.data = self._generate_data()
+
+    def _generate_data(self):
+        data = []
+        for _ in range(self.size):
+            text = "This is a sample sentence for testing purposes."
+            label = torch.tensor(0)  # Sample label
+            item = {"text": text, "label": label}
+            for name, tokenizer in self.tokenizers.items():
+                tokenized = tokenizer(text, padding="max_length", truncation=True, max_length=128)
+                item[f"input_ids_{name}"] = torch.tensor(tokenized["input_ids"])
+                item[f"attention_mask_{name}"] = torch.tensor(tokenized["attention_mask"])
+            data.append(item)
+        return data
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        return self.data[idx]
+
+
+def push_to_hub(local_dir, repo_name):
+    try:
+        repo_url = HfApi().create_repo(repo_name, exist_ok=True)
+        repo = Repository(local_dir, clone_from=repo_url)
+
+        if not os.path.exists(os.path.join(local_dir, ".git")):
+            os.system(f"cd {local_dir} && git init && git remote add origin {repo_url} && git pull origin main")
+
+        repo.git_add(auto_lfs_track=True)
+        repo.git_commit("Add model and tokenizer files")
+
+        json_files = ["config.json", "generation_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer.model", "tokenizer_config.json"]
+        for json_file in json_files:
+            json_file_path = os.path.join(local_dir, json_file)
+            if os.path.exists(json_file_path):
+                repo.git_add(json_file_path)
+
+        repo.git_push()
+        print(f"Pushed model and tokenizer to {repo_url}")
+    except Exception as e:
+        print(f"Error pushing to Hugging Face Hub: {e}")
+
+
+def main():
+    while True:
+        try:
+            os.system("git config --global credential.helper store")
+            login(token=os.getenv("HUGGINGFACE_TOKEN"), add_to_git_credential=True)
+
+            # Definir los modelos que se van a utilizar
+            models_to_train = [
+                "openai-community/gpt2-xl",
+                "google/gemma-2-9b-it",
+                "google/gemma-2-9b",
+                "meta-llama/Meta-Llama-3.1-8B-Instruct",
+                "meta-llama/Meta-Llama-3.1-8B",
+                "openbmb/MiniCPM-V-2_6",
+                "bigcode/starcoder",
+                "WizardLMTeam/WizardCoder-Python-34B-V1.0",
+                "Qwen/Qwen2-72B-Instruct",
+                "google/gemma-2-2b-it",
+                "facebook/bart-large-cnn",
+                "Falconsai/text_summarization",
+                "microsoft/speecht5_tts",
+                "Groq/Llama-3-Groq-70B-Tool-Use",
+                "Groq/Llama-3-Groq-8B-Tool-Use"
+            ]
+
+            # Inicializar los pipelines
+            pipelines_to_unify = [
+                pipeline("text-to-audio", model="facebook/musicgen-melody"),
+                pipeline("text-to-audio", model="facebook/musicgen-large"),
+                pipeline("text-to-audio", model="facebook/musicgen-small"),
+                DiffusionPipeline.from_pretrained("stabilityai/stable-video-diffusion-img2vid-xt-1-1"),
+                pipeline("automatic-speech-recognition", model="openai/whisper-small"),
+                DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-dev"),
+                DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1"),
+                DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell"),
+                pipeline("text-generation", model="meta-llama/Meta-Llama-3.1-8B"),
+                pipeline("text-generation", model="openbmb/MiniCPM-V-2_6"),
+                pipeline("text-generation", model="bigcode/starcoder"),
+                pipeline("text-to-speech", model="microsoft/speecht5_tts"),
+                pipeline("text-generation", model="WizardLMTeam/WizardCoder-Python-34B-V1.0"),
+                pipeline("text-generation", model="Qwen/Qwen2-72B-Instruct"),
+                pipeline("text-generation", model="google/gemma-2-2b-it"),
+                pipeline("summarization", model="facebook/bart-large-cnn"),
+                pipeline("summarization", model="Falconsai/text_summarization"),
+                DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-dev"),
+                pipeline("text-to-audio", model="facebook/musicgen-small"),
+                pipeline("text-generation", model="Groq/Llama-3-Groq-70B-Tool-Use"),
+                pipeline("text-generation", model="Groq/Llama-3-Groq-8B-Tool-Use")
+            ]
+
+            tokenizers = {}
+            models = []
+            for model_name in models_to_train:
+                tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+
+                if tokenizer.pad_token is None:
+                    tokenizer.add_special_tokens({'pad_token': tokenizer.eos_token})
+
+                model = AutoModel.from_pretrained(model_name)
+                tokenizers[model_name] = tokenizer
+                models.append(model)
+
+            # Agregar pipelines como modelos
+            models.extend(pipelines_to_unify)
+
+            # Crear un dataset sintético para entrenamiento y evaluación
+            synthetic_dataset = SyntheticDataset(tokenizers, size=100)
+
+            # Dividir el dataset en entrenamiento y evaluación
+            train_size = int(0.8 * len(synthetic_dataset))
+            val_size = len(synthetic_dataset) - train_size
+            train_dataset, val_dataset = torch.utils.data.random_split(synthetic_dataset, [train_size, val_size])
+
+            # Crear DataLoaders para entrenamiento y evaluación
+            train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True)
+            eval_loader = DataLoader(val_dataset, batch_size=16)
+
+            # Unificar los modelos y pipelines en uno solo
+            unified_model = UnifiedModel(models)
+            unified_model.to(torch.device("cpu"))
+
+            # Mostrar la cantidad de parámetros totales a entrenar
+            total_params = sum(p.numel() for p in unified_model.parameters())
+            print(f"Total parameters to train: {total_params}")
+
+            # Definir los argumentos de entrenamiento
+            training_args = TrainingArguments(
+                output_dir="outputs/unified_model",
+                evaluation_strategy="epoch",
+                learning_rate=9e-4,
+                per_device_train_batch_size=2,
+                per_device_eval_batch_size=16,
+                num_train_epochs=1,  # Reduced epochs for quick training
+                weight_decay=0.01,
+                logging_steps=10,  # More frequent logging for quicker feedback
+                optim="adamw_hf"
+            )
+
+            # Definir el optimizador
+            optimizer = AdamW(unified_model.parameters(), lr=training_args.learning_rate)
+
+            train_losses = []
+            eval_losses = []
+
+            def train(model, train_loader, eval_loader, args):
+                model.train()
+                epoch = 0
+                total_steps = args.num_train_epochs * len(train_loader)
+                progress_bar = tqdm(total=total_steps, desc="Training")
+
+                while epoch < args.num_train_epochs:
+                    start_time = time.time()
+                    for step, batch in enumerate(train_loader):
+                        input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
+                        attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
+                        labels = batch["label"].to("cpu")
+                        optimizer.zero_grad()
+                        outputs = model(input_ids)
+                        loss = nn.CrossEntropyLoss()(outputs, labels)
+                        loss.backward()
+                        optimizer.step()
+                        progress_bar.update(1)
+
+                        elapsed_time = time.time() - start_time
+                        estimated_total_time = total_steps * (elapsed_time / (step + 1))
+                        estimated_remaining_time = estimated_total_time - elapsed_time
+
+                        if step % args.logging_steps == 0:
+                            train_losses.append(loss.item())
+                            print(f"Step {step}/{total_steps}, Loss: {loss.item()}, Estimated remaining time: {estimated_remaining_time:.2f} seconds")
+
+                    epoch += 1
+                    model.eval()
+                    eval_loss = 0
+                    with torch.no_grad():
+                        for batch in eval_loader:
+                            input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
+                            attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
+                            labels = batch["label"].to("cpu")
+                            outputs = model(input_ids)
+                            loss = nn.CrossEntropyLoss()(outputs, labels)
+                            eval_loss += loss.item()
+
+                    eval_loss /= len(eval_loader)
+                    eval_losses.append(eval_loss)
+                    print(f"Epoch {epoch}/{args.num_train_epochs}, Evaluation Loss: {eval_loss}")
+
+            train(unified_model, train_loader, eval_loader, training_args)
+
+            # Visualizar pérdidas durante el entrenamiento
+            fig, ax = plt.subplots()
+            ax.set_xlabel("Epochs")
+            ax.set_ylabel("Loss")
+            ax.legend()
+
+            def animate(i):
+                ax.clear()
+                ax.plot(train_losses[:i], label="Train Loss")
+                ax.plot(eval_losses[:i], label="Eval Loss")
+                ax.legend()
+
+            ani = animation.FuncAnimation(fig, animate, frames=len(train_losses), blit=False)
+            plt.show()
+
+            # Subir el modelo unificado a Hugging Face Hub
+            local_dir = "./outputs/unified_model"
+            push_to_hub(local_dir, repo_name="Ffftdtd5dtft/my_model")
+
+            break
+        except Exception as e:
+            print(f"Error: {e}")
+            time.sleep(2)
+
+if __name__ == "__main__":
+    main()