app.py

#!/usr/bin/env python

import datetime
import operator
import pandas as pd
import tqdm.auto
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import HfApi
from ragatouille import RAGPretrainedModel

import gradio as gr
from gradio_calendar import Calendar
import datasets
import requests

# --- Data Loading and Processing ---

api = HfApi()

INDEX_REPO_ID = "hysts-bot-data/daily-papers-abstract-index"
INDEX_DIR_PATH = ".ragatouille/colbert/indexes/daily-papers-abstract-index/"
api.snapshot_download(
    repo_id=INDEX_REPO_ID,
    repo_type="dataset",
    local_dir=INDEX_DIR_PATH,
)
abstract_retriever = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
# Run once to initialize the retriever
abstract_retriever.search("LLM")


def update_abstract_index() -> None:
    global abstract_retriever

    api.snapshot_download(
        repo_id=INDEX_REPO_ID,
        repo_type="dataset",
        local_dir=INDEX_DIR_PATH,
    )
    abstract_retriever = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
    abstract_retriever.search("LLM")


# Scheduler for updating abstract index every hour
scheduler_abstract = BackgroundScheduler()
scheduler_abstract.add_job(
    func=update_abstract_index,
    trigger="cron",
    minute=0,  # Every hour at minute 0
    timezone="UTC",
    misfire_grace_time=3 * 60,
)
scheduler_abstract.start()


def get_df() -> pd.DataFrame:
    df = pd.merge(
        left=datasets.load_dataset("hysts-bot-data/daily-papers", split="train").to_pandas(),
        right=datasets.load_dataset("hysts-bot-data/daily-papers-stats", split="train").to_pandas(),
        on="arxiv_id",
    )
    df = df[::-1].reset_index(drop=True)
    df["date"] = pd.to_datetime(df["date"]).dt.strftime("%Y-%m-%d")

    paper_info = []
    for _, row in tqdm.auto.tqdm(df.iterrows(), total=len(df)):
        info = row.copy()
        del info["abstract"]
        info["paper_page"] = f"https://huggingface.co/papers/{row.arxiv_id}"
        paper_info.append(info)
    return pd.DataFrame(paper_info)


class Prettifier:
    @staticmethod
    def get_github_link(link: str) -> str:
        if not link:
            return ""
        return Prettifier.create_link("github", link)

    @staticmethod
    def create_link(text: str, url: str) -> str:
        return f'<a href="{url}" target="_blank">{text}</a>'

    @staticmethod
    def to_div(text: str | None, category_name: str) -> str:
        if text is None:
            text = ""
        class_name = f"{category_name}-{text.lower()}"
        return f'<div class="{class_name}">{text}</div>'

    def __call__(self, df: pd.DataFrame) -> pd.DataFrame:
        new_rows = []
        for _, row in df.iterrows():
            new_row = {
                "date": Prettifier.create_link(row.date, f"https://huggingface.co/papers?date={row.date}"),
                "paper_page": Prettifier.create_link(row.arxiv_id, row.paper_page),
                "title": row["title"],
                "github": self.get_github_link(row.github),
                "👍": row["upvotes"],
                "💬": row["num_comments"],
            }
            new_rows.append(new_row)
        return pd.DataFrame(new_rows)


class PaperList:
    COLUMN_INFO = [
        ["date", "markdown"],
        ["paper_page", "markdown"],
        ["title", "str"],
        ["github", "markdown"],
        ["👍", "number"],
        ["💬", "number"],
    ]

    def __init__(self, df: pd.DataFrame):
        self.df_raw = df
        self._prettifier = Prettifier()
        self.df_prettified = self._prettifier(df).loc[:, self.column_names]

    @property
    def column_names(self):
        return list(map(operator.itemgetter(0), self.COLUMN_INFO))

    @property
    def column_datatype(self):
        return list(map(operator.itemgetter(1), self.COLUMN_INFO))

    def search(
        self,
        start_date: datetime.datetime,
        end_date: datetime.datetime,
        title_search_query: str,
        abstract_search_query: str,
        max_num_to_retrieve: int,
    ) -> pd.DataFrame:
        df = self.df_raw.copy()
        df["date"] = pd.to_datetime(df["date"])

        # Filter by date
        df = df[(df["date"] >= start_date) & (df["date"] <= end_date)]
        df["date"] = df["date"].dt.strftime("%Y-%m-%d")

        # Filter by title
        if title_search_query:
            df = df[df["title"].str.contains(title_search_query, case=False)]

        # Filter by abstract
        if abstract_search_query:
            results = abstract_retriever.search(abstract_search_query, k=max_num_to_retrieve)
            remaining_ids = set(df["arxiv_id"])
            found_id_set = set()
            found_ids = []
            for x in results:
                arxiv_id = x["document_id"]
                if arxiv_id not in remaining_ids:
                    continue
                if arxiv_id in found_id_set:
                    continue
                found_id_set.add(arxiv_id)
                found_ids.append(arxiv_id)
            df = df[df["arxiv_id"].isin(found_ids)].set_index("arxiv_id").reindex(index=found_ids).reset_index()

        df_prettified = self._prettifier(df).loc[:, self.column_names]
        return df_prettified


# --- Sorting and Pagination Management ---

class PaperManager:
    def __init__(self, paper_list: PaperList, papers_per_page=30):
        self.paper_list = paper_list
        self.papers_per_page = papers_per_page
        self.current_page = 1
        self.sort_method = "hot"  # Default sort method
        self.total_pages = max((len(self.paper_list.df_raw) + self.papers_per_page - 1) // self.papers_per_page, 1)
        self.sort_papers()

    def calculate_score(self, row):
        """
        Calculate the score of a paper based on upvotes and age.
        This mimics the "hotness" algorithm used by platforms like Hacker News.
        """
        upvotes = row.get('👍', 0)
        published_at_str = row.get('date', datetime.datetime.now(timezone.utc).isoformat())
        try:
            published_time = datetime.datetime.strptime(published_at_str, "%Y-%m-%d")
            published_time = published_time.replace(tzinfo=datetime.timezone.utc)
        except ValueError:
            # If parsing fails, use current time to minimize the impact on sorting
            published_time = datetime.datetime.now(datetime.timezone.utc)

        time_diff = datetime.datetime.now(datetime.timezone.utc) - published_time
        time_diff_hours = time_diff.total_seconds() / 3600  # Convert time difference to hours

        # Avoid division by zero and apply the hotness formula
        score = upvotes / ((time_diff_hours + 2) ** 1.5)
        return score

    def sort_papers(self):
        df = self.paper_list.df_raw.copy()

        if self.sort_method == "hot":
            df['score'] = df.apply(self.calculate_score, axis=1)
            df_sorted = df.sort_values(by='score', ascending=False).drop(columns=['score'])
        elif self.sort_method == "new":
            df_sorted = df.sort_values(by='date', ascending=False)
        else:
            df_sorted = df

        self.paper_list.df_raw = df_sorted.reset_index(drop=True)
        self.paper_list.df_prettified = self.paper_list._prettifier(self.paper_list.df_raw).loc[:, self.paper_list.column_names]
        self.total_pages = max((len(self.paper_list.df_raw) + self.papers_per_page - 1) // self.papers_per_page, 1)
        self.current_page = 1

    def set_sort_method(self, method):
        if method not in ["hot", "new"]:
            method = "hot"
        print(f"Setting sort method to: {method}")
        self.sort_method = method
        self.sort_papers()
        return True  # Assume success

    def get_current_page_papers(self) -> str:
        start = (self.current_page - 1) * self.papers_per_page
        end = start + self.papers_per_page
        current_papers = self.paper_list.df_prettified.iloc[start:end]

        if current_papers.empty:
            return "<div class='no-papers'>No papers available for this page.</div>"

        papers_html = "".join([self.format_paper(row, idx + start + 1) for idx, row in current_papers.iterrows()])
        return f"""
        <table border="0" cellpadding="0" cellspacing="0" class="itemlist">
            {papers_html}
        </table>
        """

    def format_paper(self, row, rank):
        title = row.get('title', 'No title')
        paper_id = row.get('arxiv_id', '')
        url = f"https://huggingface.co/papers/{paper_id}"
        authors = 'Unknown'  # Assuming authors are not present in the current dataset
        upvotes = row.get('👍', 0)
        comments = row.get('💬', 0)
        published_time_str = row.get('date', datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d"))
        try:
            published_time = datetime.datetime.strptime(published_time_str, "%Y-%m-%d").replace(tzinfo=datetime.timezone.utc)
        except ValueError:
            published_time = datetime.datetime.now(datetime.timezone.utc)
        time_diff = datetime.datetime.now(datetime.timezone.utc) - published_time
        time_ago_days = time_diff.days
        time_ago = f"{time_ago_days} days ago" if time_ago_days > 0 else "today"

        return f"""
        <tr class="athing">
            <td align="right" valign="top" class="title"><span class="rank">{rank}.</span></td>
            <td valign="top" class="title">
                <a href="{url}" class="storylink" target="_blank">{title}</a>
            </td>
        </tr>
        <tr>
            <td colspan="1"></td>
            <td class="subtext">
                <span class="score">{upvotes} upvotes</span><br>
                authors: {authors} | {time_ago} | <a href="#">{comments} comments</a>
            </td>
        </tr>
        <tr style="height:5px"></tr>
        """

    def next_page(self) -> str:
        if self.current_page < self.total_pages:
            self.current_page += 1
        return self.get_current_page_papers()

    def prev_page(self) -> str:
        if self.current_page > 1:
            self.current_page -= 1
        return self.get_current_page_papers()

    def refresh(self) -> str:
        self.sort_papers()
        return self.get_current_page_papers()


# Initialize PaperList and PaperManager
def initialize_paper_manager() -> PaperManager:
    df = get_df()
    paper_list = PaperList(df)
    manager = PaperManager(paper_list)
    return manager


paper_manager = initialize_paper_manager()


def update_paper_manager() -> str:
    global paper_manager
    df = get_df()
    paper_manager.paper_list = PaperList(df)
    paper_manager.sort_papers()
    return paper_manager.get_current_page_papers()


# Scheduler for updating paper list every hour
scheduler_data = BackgroundScheduler()
scheduler_data.add_job(
    func=update_paper_manager,
    trigger="cron",
    minute=0,  # Every hour at minute 0
    timezone="UTC",
    misfire_grace_time=60,
)
scheduler_data.start()


# --- Gradio Interface Functions ---

def change_sort_method_ui(method: str) -> str:
    paper_manager.set_sort_method(method.lower())
    return paper_manager.get_current_page_papers()


def refresh_papers_ui() -> str:
    return paper_manager.refresh()


# --- CSS Styling ---

css = """
/* Existing CSS remains unchanged */
body {
    background-color: white;
    font-family: Verdana, Geneva, sans-serif;
    margin: 0;
    padding: 0;
}

a {
    color: #0000ff;
    text-decoration: none;
}

a:visited {
    color: #551A8B;
}

.container {
    width: 85%;
    margin: auto;
}

table {
    width: 100%;
}

.header-table {
    width: 100%;
    background-color: #ff6600;
    padding: 2px 10px;
}

.header-table a {
    color: black;
    font-weight: bold;
    font-size: 14pt;
    text-decoration: none;
}

.itemlist .athing {
    background-color: #f6f6ef;
}

.rank {
    font-size: 14pt;
    color: #828282;
    padding-right: 5px;
}

.storylink {
    font-size: 10pt;
}

.subtext {
    font-size: 8pt;
    color: #828282;
    padding-left: 40px;
}

.subtext a {
    color: #828282;
    text-decoration: none;
}

#refresh-button {
    background: none;
    border: none;
    color: black;
    font-weight: bold;
    font-size: 14pt;
    cursor: pointer;
}

.no-papers {
    text-align: center;
    color: #828282;
    padding: 1rem;
    font-size: 14pt;
}

@media (max-width: 640px) {
    .header-table a {
        font-size: 12pt;
    }

    .storylink {
        font-size: 9pt;
    }

    .subtext {
        font-size: 7pt;
    }
}

/* Dark mode */
@media (prefers-color-scheme: dark) {
    body {
        background-color: #121212;
        color: #e0e0e0;
    }

    a {
        color: #add8e6;
    }

    a:visited {
        color: #9370db;
    }

    .header-table {
        background-color: #ff6600;
    }

    .header-table a {
        color: black;
    }

    .itemlist .athing {
        background-color: #1e1e1e;
    }

    .rank {
        color: #b0b0b0;
    }

    .subtext {
        color: #b0b0b0;
    }

    .subtext a {
        color: #b0b0b0;
    }

    #refresh-button {
        color: #e0e0e0;
    }

    .no-papers {
        color: #b0b0b0;
    }
}
"""

# --- Initialize Gradio Blocks ---

demo = gr.Blocks(css=css)

with demo:
    with gr.Column(elem_classes=["container"]):
        # Accordion for Submission Instructions
        with gr.Accordion("How to Submit a Paper", open=False):
            gr.Markdown("""
            **Submit the paper to Daily Papers:**
            [https://huggingface.co/papers/submit](https://huggingface.co/papers/submit)

            Once your paper is submitted, it will automatically appear in this demo.
            """)

        # Header with Refresh Button
        with gr.Row():
            gr.HTML("""
            <table border="0" cellpadding="0" cellspacing="0" class="header-table">
                <tr>
                    <td>
                        <span class="pagetop">
                            <b class="hnname"><a href="#">Daily Papers</a></b>
                        </span>
                    </td>
                    <td align="right">
                        <button id="refresh-button">Refresh</button>
                    </td>
                </tr>
            </table>
            """)

        # Sort Options
        with gr.Row():
            sort_radio = gr.Radio(
                choices=["Hot", "New"],
                value="Hot",
                label="Sort By",
                interactive=True
            )

        # Paper list
        paper_list = gr.HTML()

        # Navigation Buttons
        with gr.Row():
            prev_button = gr.Button("Prev")
            next_button = gr.Button("Next")

    # Load papers on app start
    demo.load(initialize_paper_manager, outputs=[paper_list])

    # Button clicks for pagination
    prev_button.click(paper_manager.prev_page, outputs=[paper_list])
    next_button.click(paper_manager.next_page, outputs=[paper_list])

    # Refresh functionality
    refresh_button = gr.Button("Refresh", visible=False, elem_id="refresh-hidden")
    refresh_button.click(update_paper_manager, outputs=[paper_list])

    # Bind the visible Refresh button to the hidden one using JavaScript
    gr.HTML("""
    <script>
    document.getElementById('refresh-button').addEventListener('click', function() {
        document.getElementById('refresh-hidden').click();
    });
    </script>
    """)

    # Sort option change
    sort_radio.change(
        fn=change_sort_method_ui,
        inputs=[sort_radio],
        outputs=[paper_list]
    )

    # Footer
    gr.Markdown("""
    Related useful Spaces:
    - [Semantic Scholar Paper Recommender](https://huggingface.co/spaces/librarian-bots/recommend_similar_papers) by [davanstrien](https://huggingface.co/davanstrien)
    - [ArXiv CS RAG](https://huggingface.co/spaces/bishmoy/Arxiv-CS-RAG) by [bishmoy](https://huggingface.co/bishmoy)
    - [Paper Q&A](https://huggingface.co/spaces/chansung/paper_qa) by [chansung](https://huggingface.co/chansung)
    """)

# --- Launch the App ---

if __name__ == "__main__":
    demo.launch()