# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# Investigating prevalence of duplicate searches caused by filtering

# +
import os

from weco_datascience.reporting import get_recent_data
# -

df = get_recent_data(config=os.environ, n=10000,
                     index="metrics-conversion-prod")


# what time period does this start from?
df = df.sort_values("@timestamp")
df.head()

# +
# How many searches are there?

# searches=df.loc[(df(['page.name']=='images') | df(['page.name']=='works'))]
searches = df.loc[
    (df["page.name"] == "images")
    | (df["page.name"] == "works")
    & (df["page.query.query"].notnull())
    & (df["properties.totalResults"].notnull())
    & (df["page.query.page"].isnull())
]
len(searches)

# +
# How many searches unique to users are there?

searches.sort_values(
    by=[
        "session.id",
        "page.query.query",
    ]
)
dedup = searches.drop_duplicates(
    subset=["session.id", "page.query.query"], keep="first"
)
len(dedup)
# -

# Of 28,500 search events from the period 23-26/5/21, sorting by session.id and query term revealed only 11,182 
# unique session-specific queries. That's 39% of the total search figure that is used in the dashboards.

StarcoderdataJupyterScriptsDedupFiltered