added markets creator info for the tools tab

app.py

@@ -1,8 +1,6 @@
 from datetime import datetime, timedelta
 import gradio as gr
-import matplotlib.pyplot as plt
 import pandas as pd
-import seaborn as sns
 import duckdb
 import logging
 from tabs.trades import (
@@ -11,10 +9,6 @@ from tabs.trades import (
     get_overall_by_market_trades,
     get_overall_winning_trades,
     get_overall_winning_by_market_trades,
-    plot_trades_by_week,
-    plot_trades_per_market_by_week,
-    plot_winning_trades_by_week,
-    plot_winning_trades_per_market_by_week,
     integrated_plot_trades_per_market_by_week,
     integrated_plot_winning_trades_per_market_by_week,
 )
@@ -31,24 +25,20 @@ from tabs.metrics import (
 )
 
 from tabs.tool_win import (
-    get_tool_winning_rate,
+    prepare_tools,
     get_tool_winning_rate_by_market,
-    get_overall_winning_rate,
-    plot_tool_winnings_overall,
-    plot_tool_winnings_by_tool,
+    integrated_plot_tool_winnings_overall_per_market_by_week,
+    integrated_tool_winnings_by_tool_per_market,
 )
 
 from tabs.tool_accuracy import (
     plot_tools_weighted_accuracy_rotated_graph,
     plot_tools_accuracy_rotated_graph,
     compute_weighted_accuracy,
-    plot_tools_accuracy_graph,
-    plot_tools_weighted_accuracy_graph,
 )
 
 from tabs.invalid_markets import (
     plot_daily_dist_invalid_trades,
-    plot_ratio_invalid_trades_per_market,
     plot_top_invalid_markets,
     plot_daily_nr_invalid_markets,
 )
@@ -160,9 +150,7 @@ def prepare_data():
     tools_df, trades_df, tools_accuracy_info, invalid_trades = get_all_data()
     print(trades_df.info())
 
-    tools_df["request_time"] = pd.to_datetime(tools_df["request_time"])
-    trades_df["creation_timestamp"] = pd.to_datetime(trades_df["creation_timestamp"])
-
+    tools_df = prepare_tools(tools_df)
     trades_df = prepare_trades(trades_df)
 
     tools_accuracy_info = compute_weighted_accuracy(tools_accuracy_info)
@@ -184,8 +172,8 @@ demo = gr.Blocks()
 
 error_df = get_error_data(tools_df=tools_df, inc_tools=INC_TOOLS)
 error_overall_df = get_error_data_overall(error_df=error_df)
-winning_rate_df = get_tool_winning_rate(tools_df=tools_df, inc_tools=INC_TOOLS)
-winning_rate_overall_df = get_overall_winning_rate(wins_df=winning_rate_df)
+winning_df = get_tool_winning_rate_by_market(tools_df, inc_tools=INC_TOOLS)
+# preparing data for the trades graph
 trades_count_df = get_overall_trades(trades_df=trades_df)
 trades_winning_rate_df = get_overall_winning_trades(trades_df=trades_df)
 trades_by_market = get_overall_by_market_trades(trades_df=trades_df)
@@ -261,20 +249,20 @@ with demo:
             with gr.Row():
                 winning_selector = gr.Dropdown(
                     label="Select the tool metric",
-                    choices=tool_metric_choices,
+                    choices=list(tool_metric_choices.keys()),
                     value=default_tool_metric,
                 )
 
             with gr.Row():
                 # plot_tool_metrics
-                winning_plot = plot_tool_winnings_overall(
-                    wins_df=winning_rate_overall_df,
+                winning_plot = integrated_plot_tool_winnings_overall_per_market_by_week(
+                    winning_df=winning_df,
                     winning_selector=default_tool_metric,
                 )
 
             def update_tool_winnings_overall_plot(winning_selector):
-                return plot_tool_winnings_overall(
-                    wins_df=winning_rate_overall_df, winning_selector=winning_selector
+                return integrated_plot_tool_winnings_overall_per_market_by_week(
+                    winning_df=winning_df, winning_selector=winning_selector
                 )
 
             winning_selector.change(
@@ -297,12 +285,16 @@ with demo:
                 )
 
             with gr.Row():
-                tool_winnings_by_tool_plot = plot_tool_winnings_by_tool(
-                    wins_df=winning_rate_df, tool=INC_TOOLS[0]
+                tool_winnings_by_tool_plot = (
+                    integrated_tool_winnings_by_tool_per_market(
+                        wins_df=winning_df, tool=INC_TOOLS[0]
+                    )
                 )
 
             def update_tool_winnings_by_tool_plot(tool):
-                return plot_tool_winnings_by_tool(wins_df=winning_rate_df, tool=tool)
+                return integrated_tool_winnings_by_tool_per_market(
+                    wins_df=winning_df, tool=tool
+                )
 
             sel_tool.change(
                 update_tool_winnings_by_tool_plot,

scripts/markets.py

@@ -250,5 +250,33 @@ def etl(filename: Optional[str] = None) -> pd.DataFrame:
     return fpmms
 
 
+def add_market_creator(tools: pd.DataFrame) -> None:
+    # Check if fpmmTrades.parquet is in the same directory
+    try:
+        trades_filename = "fpmmTrades.parquet"
+        fpmms_trades = pd.read_parquet(DATA_DIR / trades_filename)
+    except FileNotFoundError:
+        print("Error: fpmmTrades.parquet not found. No market creator added")
+        return
+    tools["market_creator"] = ""
+    # traverse the list of traders
+    traders_list = list(tools.trader_address.unique())
+    for trader_address in traders_list:
+        market_creator = ""
+        try:
+            trades = fpmms_trades[fpmms_trades["trader_address"] == trader_address]
+            market_creator = trades.iloc[0]["market_creator"]  # first value is enough
+        except Exception:
+            print(f"ERROR getting the market creator of {trader_address}")
+            continue
+        # update
+        tools.loc[tools["trader_address"] == trader_address, "market_creator"] = (
+            market_creator
+        )
+    # filter those tools where we don't have market creator info
+    tools = tools.loc[tools["market_creator"] != ""]
+    return tools
+
+
 if __name__ == "__main__":
     etl("all_fpmms.parquet")

scripts/tools.py

@@ -45,6 +45,7 @@ from urllib3.exceptions import (
 )
 from web3 import Web3, HTTPProvider
 from web3.exceptions import MismatchedABI
+from markets import add_market_creator
 from web3.types import BlockParams
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from utils import (
@@ -586,7 +587,11 @@ def parse_store_json_events_parallel(
             contents.append(current_mech_contents)
 
     tools = pd.concat(contents, ignore_index=True)
-    print(f"Length of the contents dataframe {len(tools)}")
+    print(f"Adding market creators info. Length of the tools file = {tools}")
+    tools = add_market_creator(tools)
+    print(
+        f"Length of the tools dataframe after adding market creators info= {len(tools)}"
+    )
     print(tools.info())
     try:
         if "result" in tools.columns:

tabs/metrics.py

@@ -10,10 +10,15 @@ trade_metric_choices = [
     "ROI",
 ]
 
-tool_metric_choices = ["losses", "wins", "total_request", "win_perc"]
+tool_metric_choices = {
+    "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %": "win_perc",
+    "Total Weekly Inaccurate Nr of Mech Tool Responses": "losses",
+    "Total Weekly Accurate Nr of Mech Tool Responses": "wins",
+    "Total Weekly Nr of Mech Tool Requests": "total_request",
+}
 
 default_trade_metric = "ROI"
-default_tool_metric = "win_perc"
+default_tool_metric = "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %"
 
 HEIGHT = 600
 WIDTH = 1000

tabs/tool_win.py

@@ -1,12 +1,31 @@
 import pandas as pd
 import gradio as gr
 from typing import List
+from tabs.metrics import tool_metric_choices
+import plotly.express as px
 
 
 HEIGHT = 600
 WIDTH = 1000
 
 
+def prepare_tools(tools: pd.DataFrame) -> pd.DataFrame:
+    tools["request_time"] = pd.to_datetime(tools["request_time"])
+    tools = tools.sort_values(by="request_time", ascending=True)
+
+    tools["request_month_year_week"] = (
+        pd.to_datetime(tools["request_time"]).dt.to_period("W").dt.strftime("%b-%d")
+    )
+    # preparing the tools graph
+    # adding the total
+    tools_all = tools.copy(deep=True)
+    tools_all["market_creator"] = "all"
+    # merging both dataframes
+    tools = pd.concat([tools, tools_all], ignore_index=True)
+    tools = tools.sort_values(by="request_time", ascending=True)
+    return tools
+
+
 def get_tool_winning_rate(tools_df: pd.DataFrame, inc_tools: List[str]) -> pd.DataFrame:
     """Gets the tool winning rate data for the given tools and calculates the winning percentage."""
     tools_inc = tools_df[tools_df["tool"].isin(inc_tools)]
@@ -68,7 +87,7 @@ def get_tool_winning_rate_by_market(
     wins["total_request"] = wins[0] + wins[1]
     wins.columns = wins.columns.astype(str)
     # Convert request_month_year_week to string and explicitly set type for Altair
-    wins["request_month_year_week"] = wins["request_month_year_week"].astype(str)
+    # wins["request_month_year_week"] = wins["request_month_year_week"].astype(str)
     return wins
 
 
@@ -83,17 +102,6 @@ def get_overall_winning_rate(wins_df: pd.DataFrame) -> pd.DataFrame:
     return overall_wins
 
 
-def get_overall_winning_rate(wins_df: pd.DataFrame) -> pd.DataFrame:
-    """Gets the overall winning rate data for the given tools and calculates the winning percentage."""
-    overall_wins = (
-        wins_df.groupby("request_month_year_week")
-        .agg({"0": "sum", "1": "sum", "win_perc": "mean", "total_request": "sum"})
-        .rename(columns={"0": "losses", "1": "wins"})
-        .reset_index()
-    )
-    return overall_wins
-
-
 def get_overall_winning_rate_by_market(wins_df: pd.DataFrame) -> pd.DataFrame:
     """Gets the overall winning rate data for the given tools and calculates the winning percentage."""
     overall_wins = (
@@ -125,39 +133,68 @@ def plot_tool_winnings_overall(
     )
 
 
-def integrated_plot_tool_winnings_overall(
-    tools_df: pd.DataFrame, winning_selector: str = "win_perc"
+def sort_key(date_str):
+    month, year_week = date_str.split("-")
+    month_order = [
+        "Jan",
+        "Feb",
+        "Mar",
+        "Apr",
+        "May",
+        "Jun",
+        "Jul",
+        "Aug",
+        "Sep",
+        "Oct",
+        "Nov",
+        "Dec",
+    ]
+    month_num = month_order.index(month) + 1
+    week = int(year_week)
+    return (week // 100, month_num, week % 100)  # year, month, week
+
+
+def integrated_plot_tool_winnings_overall_per_market_by_week(
+    winning_df: pd.DataFrame,
+    winning_selector: str = "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %",
 ) -> gr.Plot:
-    # TODO Pending final implementation
-    """Plots the overall winning rate data for the given tools and calculates the winning percentage."""
-    # adding the total
-    wins_df_all = tools_df.copy(deep=True)
-    wins_df_all["market_creator"] = "all"
 
-    # merging both dataframes
-    all_winning_tools = pd.concat([wins_df, wins_df_all], ignore_index=True)
-    all_winning_tools = all_winning_tools.sort_values(
-        by="creation_timestamp", ascending=True
+    # get the column name from the metric name
+    column_name = tool_metric_choices.get(winning_selector)
+
+    wins_df = get_overall_winning_rate_by_market(winning_df)
+    # Sort the unique values of request_month_year_week
+    sorted_categories = sorted(
+        wins_df["request_month_year_week"].unique(), key=sort_key
     )
-    final_df = get_overall_winning_rate_by_market(all_winning_tools)
+    # Create a categorical type with a specific order
+    wins_df["request_month_year_week"] = pd.Categorical(
+        wins_df["request_month_year_week"], categories=sorted_categories, ordered=True
+    )
+
+    # Sort the DataFrame based on the new categorical column
+    wins_df = wins_df.sort_values("request_month_year_week")
+
     fig = px.bar(
-        final_df,
+        wins_df,
         x="request_month_year_week",
-        y=winning_selector,
+        y=column_name,
         color="market_creator",
         barmode="group",
-        color_discrete_sequence=["goldenrod", "darkgreen", "purple"],
+        color_discrete_sequence=["purple", "goldenrod", "darkgreen"],
+        category_orders={
+            "market_creator": ["pearl", "quickstart", "all"],
+            "request_month_year_week": sorted_categories,
+        },
     )
     fig.update_layout(
         xaxis_title="Week",
-        yaxis_title="Weekly % of winning rate",
+        yaxis_title=winning_selector,
         legend=dict(yanchor="top", y=0.5),
     )
     fig.update_layout(width=WIDTH, height=HEIGHT)
     fig.update_xaxes(tickformat="%b %d\n%Y")
-    return gr.Plot(
-        value=fig,
-    )
+    return gr.Plot(value=fig)
 
 
 def plot_tool_winnings_by_tool(wins_df: pd.DataFrame, tool: str) -> gr.BarPlot:
@@ -176,3 +213,44 @@ def plot_tool_winnings_by_tool(wins_df: pd.DataFrame, tool: str) -> gr.BarPlot:
         height=HEIGHT,
         width=WIDTH,
     )
+
+
+def integrated_tool_winnings_by_tool_per_market(
+    wins_df: pd.DataFrame, tool: str
+) -> gr.Plot:
+
+    tool_wins_df = wins_df[wins_df["tool"] == tool]
+    # Sort the unique values of request_month_year_week
+    sorted_categories = sorted(
+        tool_wins_df["request_month_year_week"].unique(), key=sort_key
+    )
+    # Create a categorical type with a specific order
+    tool_wins_df["request_month_year_week"] = pd.Categorical(
+        tool_wins_df["request_month_year_week"],
+        categories=sorted_categories,
+        ordered=True,
+    )
+
+    # Sort the DataFrame based on the new categorical column
+    wins_df = wins_df.sort_values("request_month_year_week")
+    fig = px.bar(
+        tool_wins_df,
+        x="request_month_year_week",
+        y="win_perc",
+        color="market_creator",
+        barmode="group",
+        color_discrete_sequence=["purple", "goldenrod", "darkgreen"],
+        category_orders={
+            "market_creator": ["pearl", "quickstart", "all"],
+            "request_month_year_week": sorted_categories,
+        },
+    )
+
+    fig.update_layout(
+        xaxis_title="Week",
+        yaxis_title="Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %",
+        legend=dict(yanchor="top", y=0.5),
+    )
+    fig.update_layout(width=WIDTH, height=HEIGHT)
+    fig.update_xaxes(tickformat="%b %d\n%Y")
+    return gr.Plot(value=fig)

tabs/trades.py

@@ -91,40 +91,6 @@ def plot_trades_by_week(trades_df: pd.DataFrame) -> gr.BarPlot:
     )
 
 
-def plot_trades_per_market_by_week(
-    trades_df: pd.DataFrame, market_type: str
-) -> gr.Plot:
-    """Plots the trades data for the given tools and calculates the winning percentage."""
-    assert "market_creator" in trades_df.columns
-    # if market_type is "all then no filter is applied"
-    if market_type == "quickstart":
-        trades = trades_df.loc[trades_df["market_creator"] == "quickstart"]
-        color_sequence = ["goldenrod"]
-
-    elif market_type == "pearl":
-        trades = trades_df.loc[trades_df["market_creator"] == "pearl"]
-        color_sequence = ["purple"]
-    else:
-        trades = trades_df
-        color_sequence = ["darkgreen"]
-
-    fig = px.bar(
-        trades,
-        x="month_year_week",
-        y="trades",
-        color_discrete_sequence=color_sequence,
-        title=market_type + " trades",
-    )
-    fig.update_layout(
-        xaxis_title="Week",
-        yaxis_title="Weekly nr of trades",
-    )
-    fig.update_xaxes(tickformat="%b %d\n%Y")
-    return gr.Plot(
-        value=fig,
-    )
-
-
 def integrated_plot_trades_per_market_by_week(trades_df: pd.DataFrame) -> gr.Plot:
 
     # adding the total