import streamlit as st from transformers import AutoTokenizer, AutoModelForCausalLM from transformers import pipeline import torch import json import pandas as pd import requests @st.cache(allow_output_mutation=True) def load_tokenizer(model_ckpt): return AutoTokenizer.from_pretrained(model_ckpt) @st.cache(allow_output_mutation=True) def load_model(model_ckpt): model = AutoModelForCausalLM.from_pretrained(model_ckpt, low_cpu_mem_usage=True) return model @st.cache() def load_examples(): with open("examples.json", "r") as f: examples = json.load(f) return examples st.set_page_config(page_icon=":laptop:", layout="wide") st.sidebar.header("Models") models = ["CodeParrot", "InCoder"] selected_models = st.sidebar.multiselect("Select code generation models to compare", models, default=["CodeParrot"]) st.sidebar.header("Tasks") tasks = [" ", "Pretraining datasets", "Model architecture", "Model evaluation", "Code generation"] selected_task = st.sidebar.selectbox("Select a task", tasks) if selected_task == " ": st.title("Code Generation Models") with open("intro.txt", "r") as f: intro = f.read() st.markdown(intro) elif selected_task == "Pretraining datasets": st.title("Pretraining datasets 📚") st.markdown("Preview of some code files from Github repositories") df = pd.read_csv("data_preview.csv") st.dataframe(df) for model in selected_models: with open(f"datasets/{model.lower()}.txt", "r") as f: text = f.read() st.markdown(f"### {model}") st.markdown(text) elif selected_task == "Model architecture": st.title("Model architecture 🔨") for model in selected_models: with open(f"architectures/{model.lower()}.txt", "r") as f: text = f.read() st.markdown(f"## {model}") st.markdown(text) st.markdown('