Update app.py

app.py

@@ -7,6 +7,7 @@ import datetime as dt
 import json
 import logging
 import sys
+import spacy
 #sys.setrecursionlimit(20000)
 import pandas as pd
 import numpy as np
@@ -16,18 +17,21 @@ from typing import Dict, List
 
 import uvicorn
 from fastapi import FastAPI, HTTPException, Request, Response
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
+from rouge_score import rouge_scorer
 import scripts.sentiment as sentiment
 import scripts.twitter_scraper as ts
+from scripts import sentiment
 from scripts.summarization import bert_summarization
 from scripts.twitter_scraper import get_latest_account_tweets
 from scripts import twitter_scraper as ts 
 import scripts.utils as utils
 from scripts import generative
 import nltk
+
 logging.basicConfig(level=logging.INFO)
 
 app = FastAPI()
@@ -57,7 +61,7 @@ async def webpage(request: Request):
 
 
 @app.get("/accounts")
-def get_accounts() -> List[dict]:
+async def get_accounts() -> List[dict]:
     import pandas as pd
 
     logging.info(f"Pulling account information on {username_list}")
@@ -94,17 +98,26 @@ def get_tweets_username(username: str) -> dict:
         print(df_tweets.head(2))
         print(df_tweets.shape)
         df_tweets = df_tweets[["handle", "created_at", "full_text"]]
-        df_tweets = df_tweets.sort_values("created_at", ascending=True).tail(10)
-        df_tweets_html = df_tweets.to_html(classes="center", index=False)
+        df_tweets["created_at"] = df_tweets["created_at"].dt.strftime("%Y-%m-%d %H:%M:%S")
+        df_tweets = df_tweets.sort_values("created_at", ascending=False)#.tail(10)
+        df_tweets_html = df_tweets.to_html(classes="center", index=False, escape=False)
+        df_tweets.to_html(open('df_tweets_html.html', 'w'))
+        df_tweets_data = df_tweets.to_dict(orient="records")
+        
+        response_data = {
+            "html": df_tweets_html,
+            "data": df_tweets_data
+        }
 
-        return HTMLResponse(content=df_tweets_html, status_code=200)
+        return JSONResponse(content=response_data, status_code=200)
+        # return HTMLResponse(content=df_tweets_html, status_code=200)
     else:
         print("Error: Failed to retrieve tweets.")
         return df_tweets
 
 
 @app.get("/audience/{username}", response_model=dict)
-def get_audience(username: str) -> dict:
+async def get_audience(username: str) -> dict:
     if username in username_list:
         query = f"from:{username} since:{start_date} until:{end_date}"
         tweets = ts.get_tweets(query=query)
@@ -203,6 +216,14 @@ async def get_sentiment(username: str) -> Dict[str, Dict[str, float]]:
 
 @app.post("/api/generate")
 async def generate_text(request: Request):
+    """Generate text from a prompt.
+
+    Args:
+        request: The HTTP request.
+
+    Returns:
+        The generated text.
+    """
     print("*" * 50)
     data = await request.json()
     print("*" * 50)
@@ -223,18 +244,18 @@ async def generate_text(request: Request):
     ###################################################
     ## Clean up generate text
     # Get rid of final sentence
-    # sentences = nltk.sent_tokenize(generated_text)
-    # unique_sentences = set()
-    # non_duplicate_sentences = []
-    # for sentence in sentences:
-    #     if sentence not in unique_sentences:
-    #         non_duplicate_sentences.append(sentence)
-    #         unique_sentences.add(sentence)
-    # final_text = " ".join(non_duplicate_sentences[:-1])
-
-    final_text= generated_text
+    sentences = nltk.sent_tokenize(generated_text)
+    unique_sentences = set()
+    non_duplicate_sentences = []
+    for sentence in sentences:
+        if sentence not in unique_sentences:
+            non_duplicate_sentences.append(sentence)
+            unique_sentences.add(sentence)
+    final_text = " ".join(non_duplicate_sentences[:-1])
+
     return {"generated_text": final_text}
 
+
 @app.post("/api/generate_summary")
 async def generate_summary(request: Request):
     """Generate summary from tweets
@@ -248,16 +269,45 @@ async def generate_summary(request: Request):
 
     print("*" * 50)
     data = await request.json()
-
+    print('data',data['tweetsData'])
     # Get the list of text
-    texts = data["text"]
+    tweets = [t['full_text'] for t in data["tweetsData"]]
 
 
-    # Generate the summary
-    summary = "This is a placeholder for summary model being returned"
+    # Concatenate tweets into a single string
+    text = " .".join(tweets)
+
+    nlp = spacy.load("en_core_web_sm")
+    nlp.add_pipe("sentencizer")
+
+    sentences = nlp(text).sents
+    # sentences = Text8Corpus(text)
+    # phrases = Phrases(
+    #     sentences, min_count=1, threshold=1, connector_words=ENGLISH_CONNECTOR_WORDS
+    # )
+    # first_sentence = next(iter(sentences))
+    # first_sentence
+    sentences = list(sentences)
+    # # Shuffle the list
+    # random.shuffle(sentences)
+    # Option 1
+    # sampled_tweets = random.sample(tweets, int(0.1 * len(tweets)))
 
+    # Option 2
+    sampled_sentences = random.sample(sentences, int(0.1 * len(sentences)))
+    
+    sampled_sentences = [sentiment.tweet_cleaner(s.text) for s in sampled_sentences]
+
+    # Join the strings into one text blob
+    tweet_blob = " ".join(sampled_sentences)
+
+    # Generate the summary
+    summary = bert_summarization(
+       tweet_blob
+    )
+    print("Summary:",summary)
     # Return the summary
-    return {"summary": summary}
+    return {"tweets_summary": summary}
 
 
 @app.get("/examples1")
@@ -272,8 +322,3 @@ async def read_examples():
     with open("templates/charts/handle_sentiment_timesteps.html") as f:
         html = f.read()
     return HTMLResponse(content=html)
-
-# uvicorn --workers=2 app:app
-# if __name__ == "__main__":
-#     # uvicorn.run(app, host="0.0.0.0", port=8000)
-#     uvicorn.run("app:app", host="127.0.0.1", port=5049, reload=True)