new non-agent graphs

app.py

@@ -12,6 +12,7 @@ from tabs.trades import (
     integrated_plot_trades_per_market_by_week,
     integrated_plot_trades_per_market_by_week_v2,
     integrated_plot_winning_trades_per_market_by_week,
+    integrated_plot_winning_trades_per_market_by_week_v2,
 )
 from tabs.staking import plot_staking_trades_per_market_by_week
 
@@ -197,19 +198,29 @@ with demo:
             with gr.Row():
                 gr.Markdown("# Trend of weekly trades")
             with gr.Row():
-                trades_by_week = integrated_plot_trades_per_market_by_week(
+                trades_by_week = integrated_plot_trades_per_market_by_week_v2(
                     trades_df=trades_df
                 )
 
             with gr.Row():
-                gr.Markdown("# Percentage of winning trades per week")
-            with gr.Row():
-                all_wtrades_by_week = integrated_plot_winning_trades_per_market_by_week(
-                    trades_df=trades_df
-                )
+                with gr.Column(scale=1):
+                    gr.Markdown("# Weekly percentage of winning for Agent based trades")
+                    agent_winning_trades = (
+                        integrated_plot_winning_trades_per_market_by_week_v2(
+                            trades_df=trades_df, trader_filter="agent"
+                        )
+                    )
+                with gr.Column(scale=1):
+                    gr.Markdown(
+                        "# Weekly percentage of winning for Non-agent based trades"
+                    )
+                    non_agent_winning_trades = (
+                        integrated_plot_winning_trades_per_market_by_week_v2(
+                            trades_df=trades_df, trader_filter="non_agent"
+                        )
+                    )
 
             def update_trade_details(trade_detail, trade_details_plot):
-                print(f"user selected option= {trade_detail}")
                 new_plot = plot_trade_metrics(
                     metric_name=trade_detail,
                     trades_df=trades_df,
@@ -217,7 +228,7 @@ with demo:
                 return new_plot
 
             with gr.Row():
-                gr.Markdown("# ⚖️ Trading metrics")
+                gr.Markdown("# ⚖️ Weekly trading metrics for all trades")
             with gr.Row():
                 trade_details_selector = gr.Dropdown(
                     label="Select a trade metric",
@@ -239,6 +250,78 @@ with demo:
                 inputs=[trade_details_selector, trade_details_plot],
                 outputs=[trade_details_plot],
             )
+
+            # Agentic traders graph
+            with gr.Row():
+                gr.Markdown("# Weekly trading metrics for trades coming from Agents")
+            with gr.Row():
+                trade_a_details_selector = gr.Dropdown(
+                    label="Select a trade metric",
+                    choices=trade_metric_choices,
+                    value=default_trade_metric,
+                )
+
+            with gr.Row():
+                with gr.Column(scale=3):
+                    a_trade_details_plot = plot_trade_metrics(
+                        metric_name=default_trade_metric,
+                        trades_df=trades_df,
+                        trader_filter="agent",
+                    )
+                with gr.Column(scale=1):
+                    trade_details_text = get_trade_metrics_text()
+
+            def update_a_trade_details(trade_detail, trade_details_plot):
+                new_a_plot = plot_trade_metrics(
+                    metric_name=trade_detail,
+                    trades_df=trades_df,
+                    trader_filter="agent",
+                )
+                return new_a_plot
+
+            trade_a_details_selector.change(
+                update_a_trade_details,
+                inputs=[trade_a_details_selector, a_trade_details_plot],
+                outputs=[a_trade_details_plot],
+            )
+
+            # Non-agentic traders graph
+            with gr.Row():
+                gr.Markdown(
+                    "# Weekly trading metrics for trades coming from Non-agents"
+                )
+            with gr.Row():
+                trade_na_details_selector = gr.Dropdown(
+                    label="Select a trade metric",
+                    choices=trade_metric_choices,
+                    value=default_trade_metric,
+                )
+
+            with gr.Row():
+                with gr.Column(scale=3):
+                    na_trade_details_plot = plot_trade_metrics(
+                        metric_name=default_trade_metric,
+                        trades_df=trades_df,
+                        trader_filter="non_agent",
+                    )
+                with gr.Column(scale=1):
+                    trade_details_text = get_trade_metrics_text()
+
+            def update_na_trade_details(trade_detail, trade_details_plot):
+                print(f"user selected option= {trade_detail}")
+                new_a_plot = plot_trade_metrics(
+                    metric_name=trade_detail,
+                    trades_df=trades_df,
+                    trader_filter="non_agent",
+                )
+                return new_a_plot
+
+            trade_na_details_selector.change(
+                update_na_trade_details,
+                inputs=[trade_na_details_selector, na_trade_details_plot],
+                outputs=[na_trade_details_plot],
+            )
+
         with gr.TabItem("🔒 Staking traders"):
             with gr.Row():
                 gr.Markdown("# Trades conducted at the Pearl markets")

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:804722496702da46e6034175b54c73778fd4c5b7794d29967dccb2f2f6432603
-size 3290989
+oid sha256:3068c54295d43c0d40f331cf3ad988fb8bf150bed0c948d3103161d7d7065f38
+size 3292156

data/outliers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9fa72b86e20493b5a02dff2cf9173d394546b5eaba1de21469bb66593f7939e1
-size 18274
+oid sha256:72326e188a845663048e6ebf368045dfc387eac9a54a38303e9020f5ca112ad6
+size 18966

scripts/profitability.py

@@ -444,15 +444,18 @@ def analyse_trader(
                 winner_trade = True
 
             # Compute mech calls
-            try:
-                num_mech_calls = (
-                    tools_usage["prompt_request"]
-                    .apply(lambda x: trade["title"] in x)
-                    .sum()
-                )
-            except Exception:
-                print(f"Error while getting the number of mech calls")
-                num_mech_calls = 2  # Average value
+            if len(tools_usage) == 0:
+                num_mech_calls = 0
+            else:
+                try:
+                    num_mech_calls = (
+                        tools_usage["prompt_request"]
+                        .apply(lambda x: trade["title"] in x)
+                        .sum()
+                    )
+                except Exception:
+                    print(f"Error while getting the number of mech calls")
+                    num_mech_calls = 2  # Average value
 
             net_earnings = (
                 earnings

scripts/pull_data.py

@@ -156,8 +156,6 @@ def only_new_weekly_analysis():
     # merge new json files with old json files
     update_json_files()
 
-    # TODO move new parquet files to a tmp folder
-
     try:
         updating_timestamps(rpc, TOOLS_FILENAME)
     except Exception as e:

scripts/update_nr_mech_calls.py

@@ -0,0 +1,60 @@
+import pandas as pd
+from profitability import DATA_DIR, DEFAULT_MECH_FEE, summary_analyse
+from tqdm import tqdm
+
+
+def update_roi(row: pd.DataFrame) -> float:
+    new_value = row.net_earnings / (
+        row.collateral_amount
+        + row.trade_fee_amount
+        + row.num_mech_calls * DEFAULT_MECH_FEE
+    )
+    return new_value
+
+
+def update_trade_nr_mech_calls(non_agents: bool = False):
+    try:
+        all_trades_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
+        tools = pd.read_parquet(DATA_DIR / "tools.parquet")
+    except Exception as e:
+        print(f"Error reading the profitability and tools parquet files")
+
+    traders = list(all_trades_df.trader_address.unique())
+    if non_agents:
+        traders = list(
+            all_trades_df.loc[
+                all_trades_df["staking"] == "non_agent"
+            ].trader_address.unique()
+        )
+
+    print("before updating")
+    print(
+        all_trades_df.loc[
+            all_trades_df["staking"] == "non_agent"
+        ].num_mech_calls.describe()
+    )
+    for trader in tqdm(traders, desc=f"Updating Traders mech calls", unit="traders"):
+        tools_usage = tools[tools["trader_address"] == trader]
+        if len(tools_usage) == 0:
+            tqdm.write(f"trader with no tools usage found {trader}")
+            all_trades_df.loc[
+                all_trades_df["trader_address"] == trader, "nr_mech_calls"
+            ] = 0
+    # update roi
+    all_trades_df["roi"] = all_trades_df.apply(lambda x: update_roi(x), axis=1)
+    print("after updating")
+    print(
+        all_trades_df.loc[
+            all_trades_df["staking"] == "non_agent"
+        ].num_mech_calls.describe()
+    )
+
+    # saving
+    all_trades_df.to_parquet(DATA_DIR / "all_trades_profitability.parquet", index=False)
+    print("Summarising trades...")
+    summary_df = summary_analyse(all_trades_df)
+    summary_df.to_parquet(DATA_DIR / "summary_profitability.parquet", index=False)
+
+
+if __name__ == "__main__":
+    update_trade_nr_mech_calls(non_agents=True)

tabs/metrics.py

@@ -118,7 +118,9 @@ def plot2_trade_details(
     )
 
 
-def plot_trade_metrics(metric_name: str, trades_df: pd.DataFrame) -> gr.Plot:
+def plot_trade_metrics(
+    metric_name: str, trades_df: pd.DataFrame, trader_filter: str = None
+) -> gr.Plot:
     """Plots the trade metrics."""
 
     if metric_name == "mech calls":
@@ -140,7 +142,16 @@ def plot_trade_metrics(metric_name: str, trades_df: pd.DataFrame) -> gr.Plot:
         column_name = metric_name
         yaxis_title = "Gross profit per trade (xDAI)"
 
-    trades_filtered = get_boxplot_metrics(column_name, trades_df)
+    if trader_filter == "agent":
+        trades_filtered = get_boxplot_metrics(
+            column_name, trades_df.loc[trades_df["staking"] != "non_agent"]
+        )
+    elif trader_filter == "non_agent":
+        trades_filtered = get_boxplot_metrics(
+            column_name, trades_df.loc[trades_df["staking"] == "non_agent"]
+        )
+    else:
+        trades_filtered = get_boxplot_metrics(column_name, trades_df)
     fig = px.box(
         trades_filtered,
         x="month_year_week",

tabs/staking.py

@@ -24,9 +24,10 @@ def plot_staking_trades_per_market_by_week(
     trades_all["market_creator"] = "all"
 
     # choose colour
-    market_colour = "darkgreen"
+
+    market_colour = "green"
     if market_creator == "pearl":
-        market_colour = "purple"
+        market_colour = "darkviolet"
     elif market_creator == "quickstart":
         market_colour = "goldenrod"
 
@@ -40,6 +41,7 @@ def plot_staking_trades_per_market_by_week(
         all_filtered_trades["market_creator"] == market_creator
     ]
     print(all_filtered_trades.market_creator.value_counts())
+
     if market_creator != "all":
         all_filtered_trades["staking"] = all_filtered_trades["staking"].replace(
             {market_creator: "staking_traders", "non_agent": "non_agent_traders"}
@@ -56,7 +58,7 @@ def plot_staking_trades_per_market_by_week(
                 "non_agent": "non_agent_traders",
             }
         )
-        colour_sequence = ["gray", "purple", "goldenrod", "black"]
+        colour_sequence = ["gray", "darkviolet", "goldenrod", "black"]
         categories_sorted = {
             "staking": [
                 "non_staking_traders",

tabs/trades.py

@@ -2,6 +2,7 @@ import gradio as gr
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
+from plotly.subplots import make_subplots
 
 
 HEIGHT = 400
@@ -70,13 +71,39 @@ def get_overall_winning_by_market_trades(trades_df: pd.DataFrame) -> pd.DataFram
         ].count()
         * 100
     )
-    # winning_trades is a series, give it a dataframe
+
     winning_trades = winning_trades.reset_index()
     winning_trades.columns = winning_trades.columns.astype(str)
     winning_trades.columns = ["month_year_week", "market_creator", "winning_trade"]
     return winning_trades
 
 
+def get_overall_winning_by_market_and_trader_type(
+    trades_df: pd.DataFrame,
+) -> pd.DataFrame:
+    """Gets the overall winning trades data for the given tools and calculates the winning percentage."""
+    # Group by week, market_creator and staking_type
+    winning_trades = (
+        trades_df.groupby(
+            ["month_year_week", "market_creator", "staking_type"], sort=False
+        )["winning_trade"].sum()
+        / trades_df.groupby(
+            ["month_year_week", "market_creator", "staking_type"], sort=False
+        )["winning_trade"].count()
+        * 100
+    )
+
+    winning_trades = winning_trades.reset_index()
+    winning_trades.columns = winning_trades.columns.astype(str)
+    winning_trades.columns = [
+        "month_year_week",
+        "market_creator",
+        "staking_type",
+        "winning_trade",
+    ]
+    return winning_trades
+
+
 def plot_trades_by_week(trades_df: pd.DataFrame) -> gr.BarPlot:
     """Plots the trades data for the given tools and calculates the winning percentage."""
     return gr.BarPlot(
@@ -150,11 +177,9 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
         .reset_index(name="trades")
     )
 
-    # category_orders={"market_creator": ["pearl", "quickstart", "all"]},
-
     # Combine the traces
     final_traces = []
-    market_colors = {"pearl": "purple", "quickstart": "goldenrod", "all": "darkgreen"}
+    market_colors = {"pearl": "darkviolet", "quickstart": "goldenrod", "all": "green"}
     # First add 'agent' traces
     for market in ["pearl", "quickstart", "all"]:
         market_data = trades[trades["market_creator"] == market]
@@ -171,6 +196,11 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
         final_traces.append(trace)
 
     # Then add 'non_agent' traces
+    market_darker_colors = {
+        "pearl": "purple",
+        "quickstart": "darkgoldenrod",
+        "all": "darkgreen",
+    }
     for market in ["pearl", "quickstart", "all"]:
         market_data = trades[trades["market_creator"] == market]
         non_agent_data = market_data[market_data["staking_type"] == "non_agent"]
@@ -179,7 +209,7 @@ def integrated_plot_trades_per_market_by_week_v2(trades_df: pd.DataFrame) -> gr.
             x=non_agent_data["month_year_week"],
             y=non_agent_data["trades"],
             name=f"{market}-non_agent",
-            marker_color="black",
+            marker_color=market_darker_colors[market],
             offsetgroup=market,
             showlegend=True,
         )
@@ -238,6 +268,55 @@ def integrated_plot_winning_trades_per_market_by_week(
     )
 
 
+def integrated_plot_winning_trades_per_market_by_week_v2(
+    trades_df: pd.DataFrame, trader_filter: str = None
+) -> gr.Plot:
+    # adding the total
+    trades_all = trades_df.copy(deep=True)
+    trades_all["market_creator"] = "all"
+
+    # merging both dataframes
+    all_filtered_trades = pd.concat([trades_df, trades_all], ignore_index=True)
+    all_filtered_trades = all_filtered_trades.sort_values(
+        by="creation_timestamp", ascending=True
+    )
+    # Create binary staking category
+    all_filtered_trades["staking_type"] = all_filtered_trades["staking"].apply(
+        lambda x: "non_agent" if x == "non_agent" else "agent"
+    )
+    if trader_filter is None:
+        final_df = get_overall_winning_by_market_trades(all_filtered_trades)
+    else:
+        final_df = get_overall_winning_by_market_and_trader_type(all_filtered_trades)
+
+    color_discrete_sequence = ["darkviolet", "goldenrod", "green"]
+    if trader_filter == "agent":
+        final_df = final_df[final_df["staking_type"] == "agent"]
+    elif trader_filter == "non_agent":
+        final_df = final_df[final_df["staking_type"] == "non_agent"]
+        color_discrete_sequence = ["purple", "darkgoldenrod", "darkgreen"]
+
+    fig = px.bar(
+        final_df,
+        x="month_year_week",
+        y="winning_trade",
+        color="market_creator",
+        barmode="group",
+        color_discrete_sequence=color_discrete_sequence,
+        category_orders={"market_creator": ["pearl", "quickstart", "all"]},
+    )
+    fig.update_layout(
+        xaxis_title="Week",
+        yaxis_title="Weekly % of winning trades",
+        legend=dict(yanchor="top", y=0.5),
+    )
+    # fig.update_layout(width=WIDTH, height=HEIGHT)
+    fig.update_xaxes(tickformat="%b %d\n%Y")
+    return gr.Plot(
+        value=fig,
+    )
+
+
 def plot_winning_trades_by_week(trades_df: pd.DataFrame) -> gr.BarPlot:
     """Plots the winning trades data for the given tools and calculates the winning percentage."""
     return gr.BarPlot(