Spaces:
Sleeping
Sleeping
| # -*- coding: utf-8 -*- | |
| """ranking_simulation_0.ipynb | |
| Automatically generated by Colaboratory. | |
| Original file is located at | |
| https://colab.research.google.com/drive/1GzjewA8ePLJ1fdk76Qax-aRRsSsNp417 | |
| """ | |
| # Commented out IPython magic to ensure Python compatibility. | |
| # %%capture | |
| # !pip install gradio | |
| # !pip install opinionated | |
| # | |
| import numpy as np | |
| import pandas as pd | |
| import opinionated | |
| import matplotlib.pyplot as plt | |
| plt.style.use("opinionated_rc") | |
| from opinionated.core import download_googlefont | |
| download_googlefont('Quicksand', add_to_cache=True) | |
| plt.rc('font', family='Quicksand') | |
| # import colormaps as cmaps | |
| def simulate_applicant_judging(num_applicants, num_judges, ratings_per_applicant, alpha, beta, | |
| judge_error=1, judgment_coarse_graining=False): | |
| """ | |
| Simulates judging of applicants by a set of judges with coarse graining functionality and random tie-breaking in ranking. | |
| :param num_applicants: Number of applicants to be judged. | |
| :param num_judges: Number of judges available for judging. | |
| :param ratings_per_applicant: Number of ratings each applicant should receive. | |
| :param alpha: Alpha parameter for the Beta distribution. | |
| :param beta: Beta parameter for the Beta distribution. | |
| :param judge_error_std_dev: Standard deviation for the judge's margin of error. | |
| :param judgment_coarse_graining: Number of buckets for coarse graining (2 to 100) or False to disable. | |
| :return: A Pandas DataFrame with detailed results for each applicant. | |
| """ | |
| # Generate the quality of applicants from a Beta distribution normalized to 0-100 | |
| applicant_qualities = np.random.beta(alpha, beta, num_applicants) * 100 | |
| # Function to apply coarse graining | |
| def coarse_grain_evaluation(evaluation, grain_size): | |
| return round(evaluation / (100 / grain_size)) * (100 / grain_size) | |
| # Initialize evaluations dictionary | |
| evaluations = {f"Applicant_{i+1}": [] for i in range(num_applicants)} | |
| judge_workload = np.zeros(num_judges) | |
| # Randomly assign judges to applicants | |
| for _ in range(ratings_per_applicant): | |
| for applicant in range(num_applicants): | |
| probabilities = (max(judge_workload) - judge_workload + 1) / sum(max(judge_workload) - judge_workload + 1) | |
| judge = np.random.choice(num_judges, p=probabilities) | |
| judge_workload[judge] += 1 | |
| evaluation = np.random.uniform(applicant_qualities[applicant]-judge_error, applicant_qualities[applicant]+judge_error) | |
| # Apply coarse graining if enabled | |
| if judgment_coarse_graining: | |
| evaluation = coarse_grain_evaluation(evaluation, judgment_coarse_graining) | |
| evaluations[f"Applicant_{applicant+1}"].append(evaluation) | |
| # Prepare data for DataFrame | |
| data = [] | |
| for applicant, (quality, scores) in enumerate(zip(applicant_qualities, evaluations.values()), 1): | |
| average_evaluation = np.mean(scores) | |
| original_ranks = np.argsort(np.argsort(-np.array(scores))) + 1 | |
| data.append([f"Applicant_{applicant}", quality, average_evaluation, scores, list(original_ranks)]) | |
| # Create DataFrame | |
| df = pd.DataFrame(data, columns=["Applicant", "Applicant Quality", "Average Evaluation", "Original Scores", "Rank of Original Scores"]) | |
| # Random tie-breaking function | |
| def random_tie_breaking(df, column): | |
| # Shuffle rows with the same value in the specified column | |
| ranks = df[column].rank(method='first', ascending=False) | |
| ties = df.duplicated(column, keep=False) | |
| shuffled_ranks = ranks[ties].sample(frac=1).sort_index() | |
| ranks[ties] = shuffled_ranks | |
| return ranks | |
| # Apply random tie-breaking to rankings | |
| df['Rank of Evaluation'] = random_tie_breaking(df, 'Average Evaluation').astype(int) | |
| df['Rank of Applicant Quality'] = random_tie_breaking(df, 'Applicant Quality').astype(int) | |
| return df | |
| # # Example usage with specified alpha and beta values | |
| # df_results = simulate_applicant_judging(num_applicants=100, num_judges=10, ratings_per_applicant=5, alpha=4, beta=4, judgment_coarse_graining=10) | |
| # df_results.head(30) # Displaying the top 30 rows for brevity | |
| # df_results.sort_values(by='Rank of Evaluation').head(30) | |
| import pandas as pd | |
| def summarize_simulation_runs(num_runs, num_applicants, num_judges, ratings_per_applicant, top_n, alpha, beta, | |
| judge_error=1, judgment_coarse_graining=False): | |
| """ | |
| Runs the applicant judging simulation multiple times and summarizes how often each candidate by quality was in the top n. | |
| :param num_runs: Number of times to run the simulation. | |
| :param num_applicants: Number of applicants to be judged. | |
| :param num_judges: Number of judges available for judging. | |
| :param ratings_per_applicant: Number of ratings each applicant should receive. | |
| :param top_n: Number of top positions to consider in the summary. | |
| :param applicant_std_dev: Standard deviation for the quality of applicants. | |
| :param judge_error_std_dev: Standard deviation for the judge's margin of error. | |
| :param judgment_coarse_graining: Number of buckets for coarse graining or False to disable. | |
| :return: A Pandas DataFrame summarizing the results. | |
| """ | |
| # Initialize counts for each quality-ranked candidate in top n positions | |
| top_n_counts = pd.DataFrame(0, index=range(1, num_applicants + 1), columns=[f'Top {i}' for i in range(1, top_n + 1)]) | |
| for _ in range(num_runs): | |
| df_results = simulate_applicant_judging(num_applicants, num_judges, ratings_per_applicant, | |
| alpha=alpha, beta=beta, judge_error=judge_error, judgment_coarse_graining=judgment_coarse_graining) | |
| # Sort by Rank of Applicant Quality | |
| sorted_by_quality = df_results.sort_values(by='Applicant Quality', ascending=False).reset_index() | |
| # Sort by Rank of Evaluation | |
| sorted_by_evaluation = df_results.sort_values(by='Rank of Evaluation').reset_index() | |
| for i in range(top_n): | |
| # Find which quality-ranked candidate is in this top evaluation position | |
| quality_rank = sorted_by_quality[sorted_by_evaluation.loc[i, 'Applicant'] == sorted_by_quality['Applicant']].index[0] + 1 | |
| top_n_counts.loc[quality_rank, f'Top {i+1}'] += 1 | |
| return top_n_counts | |
| # Example usage | |
| # num_runs = 1000 # Number of simulation runs | |
| # top_n_results = summarize_simulation_runs(num_runs=num_runs, num_applicants=100, num_judges=5, ratings_per_applicant=3, | |
| # top_n=5,alpha=2, beta=1,judge_error=4, judgment_coarse_graining=False) | |
| # top_n_results | |
| import matplotlib.pyplot as plt | |
| from matplotlib.colors import ListedColormap, LinearSegmentedColormap | |
| def plot_top_n_results(top_n_results, num_runs): | |
| """ | |
| Plots a stacked bar chart of the top-n results, filling out what's missing to num_runs in grey. | |
| :param top_n_results: DataFrame containing the counts of each quality-ranked candidate in top n positions. | |
| :param num_runs: Total number of simulation runs. | |
| """ | |
| base_cmap = cmaps.green_green1_r.cut(0.10, 'right').discrete(top_n_results.shape[1]) | |
| newcolors = base_cmap(np.linspace(0, 1, top_n_results.shape[1])) | |
| # Define a new color (pink) and add it to the colormap | |
| pink = np.array([178/256, 171/256, 165/256, 1]) | |
| newcolors = np.vstack([newcolors, pink]) | |
| # Create a new ListedColormap | |
| newcmp = ListedColormap(newcolors) | |
| # Calculate the missing counts to fill up to num_runs | |
| missing_counts = num_runs - top_n_results.sum(axis=1) | |
| # Prepare data for stacked bar chart | |
| data_to_plot = top_n_results.copy() | |
| data_to_plot['Missing'] = missing_counts | |
| # Create a figure and axis for plotting | |
| fig, ax = plt.subplots(figsize=(12, 8)) | |
| # Plot stacked bar chart | |
| data_to_plot.head(top_n_results.shape[1]).plot(kind='bar', stacked=True, colormap=newcmp, alpha=.9, ax=ax) | |
| # Plot settings | |
| ax.set_title('How often did the actually best get chosen?', loc='right') # Right-align title | |
| ax.set_xlabel('Real Applicant Rank') | |
| ax.set_ylabel('Selected in this many simulation runs') | |
| # Conditionally add legend | |
| if top_n_results.shape[1] <= 5: | |
| labels = [label.replace("Top", "Rank") for label in top_n_results.columns] + ['Not chosen'] | |
| ax.legend(labels=labels, title='Rank in Evaluation', loc='lower center', bbox_to_anchor=(0.5, -0.2), ncol=top_n_results.shape[1]+1) # Legend below the chart | |
| plt.tight_layout() | |
| return fig | |
| # plt.show() | |
| # plot = plot_top_n_results(top_n_results, num_runs) | |
| # num_applicants=100 | |
| # num_judges=10 | |
| # ratings_per_applicant=5 | |
| # top_n=5 | |
| # applicant_std_dev=20 | |
| # judge_error_std_dev=1 | |
| # judgment_coarse_graining=6 | |
| # top_n_counts = pd.DataFrame(0, index=range(1, num_applicants + 1), columns=[f'Top {i}' for i in range(1, top_n + 1)]) | |
| # df_results = simulate_applicant_judging(num_applicants, num_judges, ratings_per_applicant, | |
| # applicant_std_dev, judge_error_std_dev, judgment_coarse_graining) | |
| # display(df_results.sort_values(by='Rank of Evaluation').head(10)) | |
| # # Sort by Rank of Applicant Quality | |
| # sorted_by_quality = df_results.sort_values(by='Applicant Quality', ascending=False).reset_index() | |
| # # Sort by Rank of Evaluation | |
| # sorted_by_evaluation = df_results.sort_values(by='Rank of Evaluation').reset_index() | |
| # for i in range(top_n): | |
| # # Find which quality-ranked candidate is in this top evaluation position | |
| # quality_rank = sorted_by_quality[sorted_by_evaluation.loc[i, 'Applicant'] == sorted_by_quality['Applicant']].index[0] + 1 | |
| # top_n_counts.loc[quality_rank, f'Top {i+1}'] += 1 | |
| # display(top_n_counts.head(15)) | |
| # import numpy as np | |
| # import matplotlib.pyplot as plt | |
| # import seaborn as sns | |
| # from scipy.stats import gaussian_kde | |
| # def visualize_applicant_and_judge_distributions(alpha, beta, judge_error=1): | |
| # """ | |
| # Visualizes the distribution of applicants' qualities and an example of a judge's evaluations for a random applicant | |
| # using density estimates with normalized heights. | |
| # :param applicant_std_dev: Standard deviation for the quality of applicants. | |
| # :param judge_error_std_dev: Standard deviation for the judge's margin of error. | |
| # """ | |
| # # Generate applicant qualities | |
| # applicant_qualities = np.random.beta(alpha, beta, 5000) * 100 | |
| # # Choose a random applicant for judge's evaluation | |
| # random_applicant_quality = np.random.choice(applicant_qualities) | |
| # judge_evaluations = np.random.normal(random_applicant_quality, judge_error, 5000) | |
| # # Calculate KDEs and find peak values | |
| # kde_applicant = gaussian_kde(applicant_qualities) | |
| # kde_judge = gaussian_kde(judge_evaluations) | |
| # x = np.linspace(0, 100, 1000) | |
| # kde_applicant_vals = kde_applicant(x) | |
| # kde_judge_vals = kde_judge(x) | |
| # peak_applicant = np.max(kde_applicant_vals) | |
| # peak_judge = np.max(kde_judge_vals) | |
| # scale_factor = peak_applicant / peak_judge | |
| # # Plotting | |
| # plt.figure(figsize=(12, 6)) | |
| # # Plot for distribution of all applicants | |
| # sns.lineplot(x=x, y=kde_applicant_vals, color="blue", label='Applicant Qualities') | |
| # plt.fill_between(x, kde_applicant_vals, color="blue", alpha=0.3) | |
| # plt.title('Distribution of Applicant Qualities') | |
| # plt.xlabel('Quality') | |
| # plt.ylabel('Normalized Density') | |
| # plt.legend() | |
| # plt.xlim(0, 100) | |
| # # # Plot for distribution of a single applicant's evaluations | |
| # # sns.lineplot(x=x, y=kde_judge_vals * scale_factor, color='orange', label='Judge Evaluations') | |
| # # plt.fill_between(x, kde_judge_vals * scale_factor, color="orange", alpha=0.3) | |
| # # plt.title('Distribution of a Judge\'s Evaluations for a Chosen Applicant') | |
| # # plt.xlabel('Evaluation Score') | |
| # # plt.ylabel('Normalized Density') | |
| # # plt.legend() | |
| # # plt.xlim(0, 100) | |
| # plt.tight_layout() | |
| # plt.show() | |
| # # Example usage | |
| # visualize_applicant_and_judge_distributions(alpha=2, beta=1, judge_error=5) | |
| import gradio as gr | |
| import matplotlib.pyplot as plt | |
| from io import BytesIO | |
| from scipy.stats import beta | |
| # Function to plot beta distribution | |
| def plot_beta_distribution(a, b, judgement_variability): | |
| x = np.linspace(0, 1, 1000) | |
| y = beta.pdf(x, a, b) | |
| fig, ax = plt.subplots(figsize=(7, 3)) # Figure size | |
| plt.fill_between(np.linspace(0, 100, 1000), y, color="#ee4d5a", alpha=0.8) | |
| # plt.title('Distribution of Applicant Qualities') | |
| plt.xlabel('True Applicants Quality-Distribution') | |
| plt.xlim(0, 100) | |
| ax.set_yticklabels([]) # Remove y-axis labels | |
| # Drawing the line | |
| line_length = 2 * judgement_variability | |
| line_x = [50 - line_length/2, 50 + line_length/2] | |
| plt.plot(line_x, [0, 0], color='black', linewidth=2) | |
| # Labeling the line | |
| plt.text(np.mean(line_x), 0.02, 'Judgement Variability', ha='center', va='bottom', color='black') | |
| return fig | |
| # Your existing function for running simulation and plotting | |
| def run_simulation_and_plot(num_runs, num_applicants, num_judges, ratings_per_applicant, top_n, alpha, beta, judge_error, judgment_coarse_graining,judgment_coarse_graining_true_false): | |
| if judgment_coarse_graining_true_false == False: | |
| judgment_coarse_graining = False | |
| top_n_results = summarize_simulation_runs(num_runs, num_applicants, num_judges, ratings_per_applicant, top_n, alpha, beta, judge_error, judgment_coarse_graining) | |
| return plot_top_n_results(top_n_results, num_runs) | |
| intro_html = """ | |
| <h1>On Rankings</h1> | |
| <p>One of the central experiences of being an academic is the experience of being ranked. We are ranked when we apply for graduate school, or maybe already for a master's degree. We are ranked when we're up for faculty positions. We are ranked when we submit abstracts for conferences. And when we publish papers, we do so, of course, in journals that are ranked. The places where we work, the departments, are ranked, of course, as well. But although rankings apparently are catnip to academics, and probably everybody else as well, we do have some agreement. Most people probably share the intuition that there's something weird or iffy about rankings, and the suspicion that maybe often they are not as informative as there are some beings absolutely everywhere who suggest.</p> | |
| """ | |
| comment_distribution_image = """<p>This is the disrtribution from which our applicants will be sampled:</p>""" | |
| # Building the interface | |
| with gr.Blocks(theme=gr.themes.Monochrome()) as demo: | |
| with gr.Column(): | |
| gr.HTML(intro_html) | |
| with gr.Row(): | |
| with gr.Column(): | |
| run_button = gr.Button("Run Simulations!") | |
| # control applicant distribution | |
| with gr.Group(): | |
| alpha_slider = gr.Slider(0.1, 5, step=0.1, value=1.4, label="Alpha (β Distribution)") | |
| beta_slider = gr.Slider(0.1, 5, step=0.1,value=2.7, label="Beta (β Distribution)") | |
| # simumlation-settings: | |
| num_applicants = gr.Slider(10, 300, step=10, value=100, label="Number of Applicants") | |
| num_judges = gr.Slider(1, 100, step=1, value=7, label="Number of Judges") | |
| ratings_per_applicant = gr.Slider(1, 5, step=1, value=3, label="Ratings per Applicant", info='how many different ratings each application gets.') | |
| top_n = gr.Slider(1, 40, step=1, value=5, label="Top N") | |
| judge_error = gr.Slider(0, 10, step=1, value=2, label="Judge Error") | |
| judgment_coarse_graining_true_false = gr.Checkbox(value= True, label="Coarse grain judgements.") | |
| judgment_coarse_graining = gr.Slider(0, 30, step=1, value=7, label="Coarse Graining Factor") | |
| num_runs = gr.Slider(10, 1000, step=10,value=100, label="Number of Runs") | |
| # The button to run the simulation | |
| # Sliders for alpha and beta parameters of the beta distribution | |
| with gr.Column(): | |
| with gr.Group(): | |
| beta_plot = gr.Plot(label="Applicants quality distribution") | |
| gr.HTML(comment_distribution_image) | |
| # Your existing plot output | |
| plot_output = gr.Plot(label="Simulation Results",show_label=True) | |
| #https://discuss.huggingface.co/t/gradio-changing-background-colour-in-all-devices/42519 | |
| # Function call on button click | |
| run_button.click( | |
| run_simulation_and_plot, | |
| inputs=[num_runs, num_applicants, num_judges, ratings_per_applicant, top_n, alpha_slider, beta_slider, judge_error, judgment_coarse_graining,judgment_coarse_graining_true_false], | |
| outputs=[plot_output] | |
| ) | |
| alpha_slider.change(plot_beta_distribution, inputs=[alpha_slider, beta_slider,judge_error], outputs=[beta_plot]) | |
| beta_slider.change(plot_beta_distribution, inputs=[alpha_slider, beta_slider,judge_error], outputs=[beta_plot]) | |
| judge_error.change(plot_beta_distribution, inputs=[alpha_slider, beta_slider,judge_error], outputs=[beta_plot]) | |
| demo.load(plot_beta_distribution, inputs=[alpha_slider, beta_slider,judge_error], outputs=[beta_plot]) | |
| if __name__ == "__main__": | |
| demo.launch(debug=True) | |