app.py

#!/usr/bin/env python

import datetime
import operator
import pandas as pd
import tqdm.auto
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import HfApi

import gradio as gr
from gradio_calendar import Calendar
import datasets
import requests

from datetime import timezone  # Ensure timezone is imported

# --- Data Loading and Processing ---

api = HfApi()

INDEX_REPO_ID = "hysts-bot-data/daily-papers-abstract-index"
INDEX_DIR_PATH = ".ragatouille/colbert/indexes/daily-papers-abstract-index/"

# Removed ragatouille and abstract_retriever initialization
# If INDEX_REPO_ID is not used elsewhere, consider removing related lines

# Removed abstract_retriever initialization and search

def update_abstract_index() -> None:
    """
    Removed abstract_retriever update functionality since ragatouille is no longer used.
    """
    pass  # No operation needed

# Scheduler for updating abstract index every hour
# Removed scheduler_abstract as it's no longer necessary
# If INDEX_REPO_ID is not used elsewhere, consider removing the download

# Optionally, remove the snapshot_download if the index is not needed
# api.snapshot_download(
#     repo_id=INDEX_REPO_ID,
#     repo_type="dataset",
#     local_dir=INDEX_DIR_PATH,
# )

# --- DataFrame Preparation ---

def get_df() -> pd.DataFrame:
    # Load and merge datasets
    df_papers = datasets.load_dataset("hysts-bot-data/daily-papers", split="train").to_pandas()
    df_stats = datasets.load_dataset("hysts-bot-data/daily-papers-stats", split="train").to_pandas()
    df = pd.merge(left=df_papers, right=df_stats, on="arxiv_id")
    
    # Reverse the DataFrame to have the latest papers first
    df = df[::-1].reset_index(drop=True)
    
    # Ensure 'date' is in datetime format and handle missing dates
    df["date"] = pd.to_datetime(df["date"], errors='coerce')
    df["date"] = df["date"].dt.strftime("%Y-%m-%d").fillna(datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d"))
    
    # Prepare the DataFrame by removing 'abstract' and adding 'paper_page'
    paper_info = []
    for _, row in tqdm.auto.tqdm(df.iterrows(), total=len(df)):
        info = row.copy()
        if "abstract" in info:
            del info["abstract"]
        paper_info.append(info)
    df_prepared = pd.DataFrame(paper_info)
    
    # Add 'paper_page' links
    df_prepared["paper_page"] = df_prepared["arxiv_id"].apply(lambda x: f"https://huggingface.co/papers/{x}")
    
    return df_prepared


class Prettifier:
    @staticmethod
    def get_github_link(link: str) -> str:
        if not link:
            return ""
        return Prettifier.create_link("github", link)

    @staticmethod
    def create_link(text: str, url: str) -> str:
        return f'<a href="{url}" target="_blank">{text}</a>'

    def __call__(self, df: pd.DataFrame) -> pd.DataFrame:
        new_rows = []
        for _, row in df.iterrows():
            # Handle published_at: original date
            published_at = row["date"]  # Already formatted as "%Y-%m-%d"
            
            # Handle date link
            date_display = Prettifier.create_link(row.date, f"https://huggingface.co/papers?date={row.date}")
            
            new_row = {
                "arxiv_id": row["arxiv_id"],  # Include arxiv_id
                "date_display": date_display,  # For display
                "published_at": published_at,  # For internal calculations
                "paper_page": Prettifier.create_link(row.arxiv_id, row.paper_page),
                "title": row["title"],
                "github": Prettifier.get_github_link(row.get("github", "")),
                "👍": row["upvotes"],
                "💬": row["num_comments"],
            }
            new_rows.append(new_row)
        return pd.DataFrame(new_rows)


class PaperList:
    COLUMN_INFO = [
        ["arxiv_id", "str"],         # Added arxiv_id
        ["date_display", "markdown"],# For display
        ["published_at", "str"],     # For internal use
        ["paper_page", "markdown"],
        ["title", "str"],
        ["github", "markdown"],
        ["👍", "number"],
        ["💬", "number"],
    ]

    def __init__(self, df: pd.DataFrame):
        self.df_raw = df
        self._prettifier = Prettifier()
        self.df_prettified = self._prettifier(df).loc[:, self.column_names]

    @property
    def column_names(self):
        return list(map(operator.itemgetter(0), self.COLUMN_INFO))

    @property
    def column_datatype(self):
        return list(map(operator.itemgetter(1), self.COLUMN_INFO))

    def search(
        self,
        title_search_query: str,
        max_num_to_retrieve: int = 1000,  # Set a high default to include all if not specified
    ) -> pd.DataFrame:
        df = self.df_raw.copy()

        # Filter by title if search query is provided
        if title_search_query:
            df = df[df["title"].str.contains(title_search_query, case=False, na=False)]
        
        # Limit the number of papers to retrieve if max_num_to_retrieve is set
        if max_num_to_retrieve:
            df = df.head(max_num_to_retrieve)

        # Prettify the DataFrame
        df_prettified = self._prettifier(df).loc[:, self.column_names]
        return df_prettified


# --- Sorting and Pagination Management ---

class PaperManager:
    def __init__(self, paper_list: PaperList, papers_per_page=30):
        self.paper_list = paper_list
        self.papers_per_page = papers_per_page
        self.sort_method = "hot"  # Default sort method
        self.current_search_query = ""  # Initialize with no search query
        self.sort_papers()
        # 'current_page' and 'total_pages' are set in 'sort_papers()'

    def calculate_score(self, row):
        """
        Calculate the score of a paper based on upvotes and age.
        This mimics the "hotness" algorithm used by platforms like Hacker News.
        """
        upvotes = row.get('👍', 0)
        published_at_str = row.get('published_at', datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d"))  # **FIX** Changed from 'date' to 'published_at'
        try:
            published_time = datetime.datetime.strptime(published_at_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        except ValueError:
            # If parsing fails, use current time to minimize the impact on sorting
            published_time = datetime.datetime.now(timezone.utc)

        time_diff = datetime.datetime.now(timezone.utc) - published_time
        time_diff_hours = time_diff.total_seconds() / 3600  # Convert time difference to hours

        # Avoid division by zero and apply the hotness formula
        score = upvotes / ((time_diff_hours + 2) ** 1.5) if (time_diff_hours + 2) > 0 else 0
        return score

    def sort_papers(self):
        df = self.paper_list.df_raw.copy()

        # Apply search filter if a search query exists
        if self.current_search_query:
            df = df[df["title"].str.contains(self.current_search_query, case=False, na=False)]

        if self.sort_method == "hot":
            df['score'] = df.apply(self.calculate_score, axis=1)
            df_sorted = df.sort_values(by='score', ascending=False).drop(columns=['score'])
        elif self.sort_method == "new":
            df_sorted = df.sort_values(by='published_at', ascending=False)  # **FIX** Changed from 'date' to 'published_at'
        else:
            df_sorted = df

        self.paper_list.df_raw = df_sorted.reset_index(drop=True)
        self.paper_list.df_prettified = self.paper_list._prettifier(self.paper_list.df_raw).loc[:, self.paper_list.column_names]
        self.total_pages = max((len(self.paper_list.df_raw) + self.papers_per_page - 1) // self.papers_per_page, 1)
        self.current_page = 1

    def set_sort_method(self, method):
        if method not in ["hot", "new"]:
            method = "hot"
        print(f"Setting sort method to: {method}")
        self.sort_method = method
        self.sort_papers()
        return True  # Assume success

    def set_search_query(self, query: str):
        print(f"Setting search query to: {query}")
        self.current_search_query = query
        self.sort_papers()
        return True  # Assume success

    def get_current_page_papers(self) -> str:
        start = (self.current_page - 1) * self.papers_per_page
        end = start + self.papers_per_page
        current_papers = self.paper_list.df_prettified.iloc[start:end]

        if current_papers.empty:
            return "<div class='no-papers'>No papers available for this page.</div>"

        papers_html = "".join([self.format_paper(row, idx + start + 1) for idx, row in current_papers.iterrows()])
        return f"""
        <table border="0" cellpadding="0" cellspacing="0" class="itemlist">
            {papers_html}
        </table>
        """

    def format_paper(self, row, rank):
        title = row.get('title', 'No title')
        paper_id = row.get('arxiv_id', '')
        url = f"https://huggingface.co/papers/{paper_id}"
        upvotes = row.get('👍', 0)
        comments = row.get('💬', 0)
        published_time_str = row.get('published_at', datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d"))
        try:
            published_time = datetime.datetime.strptime(published_time_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        except ValueError:
            published_time = datetime.datetime.now(timezone.utc)
        time_diff = datetime.datetime.now(timezone.utc) - published_time
        time_ago_days = time_diff.days
        time_ago = f"{time_ago_days} days ago" if time_ago_days > 0 else "today"

        return f"""
        <tr class="athing">
            <td align="right" valign="top" class="title"><span class="rank">{rank}.</span></td>
            <td valign="top" class="title">
                <a href="{url}" class="storylink" target="_blank">{title}</a>
            </td>
        </tr>
        <tr>
            <td colspan="1"></td>
            <td class="subtext">
                <span class="score">{upvotes} upvotes</span><br>
                {time_ago} | <a href="#">{comments} comments</a>
            </td>
        </tr>
        <tr style="height:5px"></tr>
        """

    def next_page(self) -> str:
        if self.current_page < self.total_pages:
            self.current_page += 1
        return self.get_current_page_papers()

    def prev_page(self) -> str:
        if self.current_page > 1:
            self.current_page -= 1
        return self.get_current_page_papers()

    def refresh(self) -> str:
        self.sort_papers()
        return self.get_current_page_papers()


# Initialize PaperList and PaperManager
def initialize_paper_manager() -> str:
    df = get_df()
    paper_list = PaperList(df)
    manager = PaperManager(paper_list)
    return manager.get_current_page_papers()  # Return HTML string instead of the manager object


paper_manager = None  # Initialize globally


def setup_paper_manager():
    global paper_manager
    df = get_df()
    paper_list = PaperList(df)
    paper_manager = PaperManager(paper_list)


# Initialize PaperManager at the start
setup_paper_manager()


def update_paper_manager() -> str:
    global paper_manager
    df = get_df()
    paper_manager.paper_list = PaperList(df)
    paper_manager.sort_papers()
    return paper_manager.get_current_page_papers()


# Scheduler for updating paper list every hour
scheduler_data = BackgroundScheduler()
scheduler_data.add_job(
    func=update_paper_manager,
    trigger="cron",
    minute=0,  # Every hour at minute 0
    timezone="UTC",
    misfire_grace_time=60,
)
scheduler_data.start()


# --- Gradio Interface Functions ---

def change_sort_method_ui(method: str) -> str:
    paper_manager.set_sort_method(method.lower())
    return paper_manager.get_current_page_papers()


def refresh_papers_ui() -> str:
    return paper_manager.refresh()


def search_papers_ui(query: str) -> str:
    paper_manager.set_search_query(query)
    return paper_manager.get_current_page_papers()


# --- CSS Styling ---

css = """
/* Existing CSS remains unchanged */
body {
    background-color: white;
    font-family: Verdana, Geneva, sans-serif;
    margin: 0;
    padding: 0;
}

a {
    color: #0000ff;
    text-decoration: none;
}

a:visited {
    color: #551A8B;
}

.container {
    width: 85%;
    margin: auto;
}

table {
    width: 100%;
}

.header-table {
    width: 100%;
    background-color: #ff6600;
    padding: 2px 10px;
}

.header-table a {
    color: black;
    font-weight: bold;
    font-size: 14pt;
    text-decoration: none;
}

.itemlist .athing {
    background-color: #f6f6ef;
}

.rank {
    font-size: 14pt;
    color: #828282;
    padding-right: 5px;
}

.storylink {
    font-size: 10pt;
}

.subtext {
    font-size: 8pt;
    color: #828282;
    padding-left: 40px;
}

.subtext a {
    color: #828282;
    text-decoration: none;
}

#refresh-button {
    background: none;
    border: none;
    color: black;
    font-weight: bold;
    font-size: 14pt;
    cursor: pointer;
}

.no-papers {
    text-align: center;
    color: #828282;
    padding: 1rem;
    font-size: 14pt;
}

@media (max-width: 640px) {
    .header-table a {
        font-size: 12pt;
    }

    .storylink {
        font-size: 9pt;
    }

    .subtext {
        font-size: 7pt;
    }
}

/* Dark mode */
@media (prefers-color-scheme: dark) {
    body {
        background-color: #121212;
        color: #e0e0e0;
    }

    a {
        color: #add8e6;
    }

    a:visited {
        color: #9370db;
    }

    .header-table {
        background-color: #ff6600;
    }

    .header-table a {
        color: black;
    }

    .itemlist .athing {
        background-color: #1e1e1e;
    }

    .rank {
        color: #b0b0b0;
    }

    .subtext {
        color: #b0b0b0;
    }

    .subtext a {
        color: #b0b0b0;
    }

    #refresh-button {
        color: #e0e0e0;
    }

    .no-papers {
        color: #b0b0b0;
    }
}
"""


# --- Initialize Gradio Blocks ---

demo = gr.Blocks(css=css)

with demo:
    with gr.Column(elem_classes=["container"]):
        # Accordion for Submission Instructions
        with gr.Accordion("How to Submit a Paper", open=False):
            gr.Markdown("""
            **Submit the paper to Daily Papers:**
            [https://huggingface.co/papers/submit](https://huggingface.co/papers/submit)

            Once your paper is submitted, it will automatically appear in this demo.
            """)
        # Header with Refresh Button
        with gr.Row():
            gr.HTML("""
            <table border="0" cellpadding="0" cellspacing="0" class="header-table">
                <tr>
                    <td>
                        <span class="pagetop">
                            <b class="hnname"><a href="#">Daily Papers</a></b>
                        </span>
                    </td>
                    <td align="right">
                        <button id="refresh-button">Refresh</button>
                    </td>
                </tr>
            </table>
            """)
        # Search Bar and Sort Options
        with gr.Row():
            search_box = gr.Textbox(
                label="Search Papers by Title",
                placeholder="Enter keywords to search...",
                lines=1,
                interactive=True
            )
            search_button = gr.Button("Search")
        with gr.Row():
            sort_radio = gr.Radio(
                choices=["Hot", "New"],
                value="Hot",
                label="Sort By",
                interactive=True
            )
        # Paper list
        paper_list = gr.HTML()
        # Navigation Buttons
        with gr.Row():
            prev_button = gr.Button("Prev")
            next_button = gr.Button("Next")

    # Load papers on app start
    demo.load(
        fn=lambda: paper_manager.get_current_page_papers(),
        outputs=[paper_list]
    )

    # Button clicks for pagination
    prev_button.click(paper_manager.prev_page, outputs=[paper_list])
    next_button.click(paper_manager.next_page, outputs=[paper_list])

    # Refresh functionality
    refresh_button = gr.Button("Refresh", visible=False, elem_id="refresh-hidden")
    refresh_button.click(update_paper_manager, outputs=[paper_list])

    # Bind the visible Refresh button to the hidden one using JavaScript
    gr.HTML("""
    <script>
    document.getElementById('refresh-button').addEventListener('click', function() {
        document.getElementById('refresh-hidden').click();
    });
    </script>
    """)

    # Sort option change
    sort_radio.change(
        fn=change_sort_method_ui,
        inputs=[sort_radio],
        outputs=[paper_list]
    )

    # Search functionality
    search_button.click(
        fn=search_papers_ui,
        inputs=[search_box],
        outputs=[paper_list]
    )

    # Footer
    gr.Markdown("""
    Related useful Spaces:
    - [Semantic Scholar Paper Recommender](https://huggingface.co/spaces/librarian-bots/recommend_similar_papers) by [davanstrien](https://huggingface.co/davanstrien)
    - [ArXiv CS RAG](https://huggingface.co/spaces/bishmoy/Arxiv-CS-RAG) by [bishmoy](https://huggingface.co/bishmoy)
    - [Paper Q&A](https://huggingface.co/spaces/chansung/paper_qa) by [chansung](https://huggingface.co/chansung)
    """)


# --- Launch the App ---

if __name__ == "__main__":
    demo.launch()