Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import pandas as pd | |
| from datasets import load_dataset | |
| # Initialize total counts | |
| total_yes_count = 0 | |
| total_count = 0 | |
| def calculate_percentage(*answers): | |
| """ | |
| Calculate the percentage of 'yes' answers. | |
| :param answers: Iterable containing the answers. | |
| :returns: Percentage of 'yes' answers. | |
| """ | |
| global total_yes_count, total_count | |
| yes_count = sum(answers) | |
| total_yes_count += yes_count | |
| total_count += len(answers) | |
| percentage = yes_count / len(answers) * 100 | |
| return f"{percentage}%" | |
| def calculate_overall_percentage(): | |
| """ | |
| Calculate the overall percentage of 'yes' answers. | |
| :returns: Overall percentage of 'yes' answers. | |
| """ | |
| global total_yes_count, total_count | |
| if total_count != 100: | |
| return "Make sure you have submitted your answers in all the tabs." | |
| overall_percentage = total_yes_count | |
| return f"{overall_percentage}%" | |
| # Load data | |
| dataset = load_dataset("mariagrandury/fmti-indicators") | |
| df = pd.DataFrame(dataset["train"]) | |
| grouped = df.groupby(["Domain", "Subdomain"]) | |
| # Create an interface per group of indicators | |
| interfaces = [] | |
| tab_names = [] | |
| for (domain, subdomain), group in grouped: | |
| questions = group["Definition"].tolist() | |
| inputs = [gr.Checkbox(label=question) for question in questions] | |
| output = gr.Textbox(label="Subdomain Percentage") | |
| iface = gr.Interface( | |
| fn=calculate_percentage, | |
| inputs=inputs, | |
| outputs=output, | |
| title=f"{domain} - {subdomain}", | |
| allow_flagging="never", | |
| ) | |
| interfaces.append(iface) | |
| tab_names.append(subdomain) | |
| # Add overall percentage button | |
| overall_button = gr.Interface( | |
| fn=calculate_overall_percentage, | |
| inputs=[], | |
| outputs=gr.Textbox(label="Overall Percentage"), | |
| title="Transparency Score", | |
| allow_flagging="never", | |
| ) | |
| interfaces.append(overall_button) | |
| tab_names.append("Total Transparency Score") | |
| # Create the tabbed interface | |
| tabbed_interface = gr.TabbedInterface( | |
| interface_list=interfaces, | |
| tab_names=tab_names, | |
| title="The Foundation Model Transparency Index", | |
| ) | |
| # Combine blocks to create demo | |
| with gr.Blocks(title="FMTI") as demo: | |
| gr.Markdown( | |
| """ | |
| # Transparency Self-Assessment (FMTI) | |
| This tool allows you to self-assess the transparency of your model based on the Foundation Model Transparency Index developed by the Center for Research on Foundation Models. | |
| """ | |
| ) | |
| with gr.Accordion(label="Instructions", open=True): | |
| gr.Markdown( | |
| """ | |
| The FMTI defines 100 indicators that characterize transparency for foundation model developers. They are divided into three broad domains: "Upstream" (model building), "Model" (model properties and function) and "Downstream" (model distribution). In addition to these top-level domains, the indicators are also grouped together into subdomains. | |
| Each tab below contains yes-or-no questions for each subdomain. Read all questions and check the boxes corresponding to the 'yes' responses. "Submit" your answers before proceeding to the next tab. Upon reaching the final tab, "Total Transparency Score", click on "Generate" to compute your model's overall transparency score. | |
| More info about the FMTI at https://crfm.stanford.edu/fmti/. | |
| Please note: this tool is research work and NOT a commercial or legal product. | |
| """ | |
| ) | |
| gr.TabbedInterface( | |
| interface_list=interfaces, | |
| tab_names=tab_names, | |
| title="", | |
| ) | |
| gr.Markdown( | |
| """ | |
| ## Compare your results | |
| How transparent is your model compared to the ones in the 2023 study? Check the graphics below! | |
| Images source: https://crfm.stanford.edu/fmti | |
| """ | |
| ) | |
| with gr.Row(): | |
| gr.Image( | |
| "https://crfm.stanford.edu/fmti/fmti-flagship.jpg", | |
| show_label=False, | |
| show_download_button=False, | |
| ) | |
| gr.Image( | |
| "https://crfm.stanford.edu/fmti/subdomain-scores.png", | |
| show_label=False, | |
| show_download_button=False, | |
| ) | |
| demo.launch() | |