feat(app): added scraper thanks to @leondz

app.py

@@ -1,12 +1,14 @@
 import gradio as gr
+import urllib.request
 from datetime import date
+from bs4 import BeautifulSoup
 
 from avidtools.datamodels.report import Report
 from avidtools.datamodels.components import *
 from avidtools.datamodels.enums import *
 
 # def generate_report():
-def generate_report(classof,type,risk_domain,sep,lifecycle):
+def generate_report(uri,title,abstract,classof,type,risk_domain,sep,lifecycle):
 # def generate_report(scraped_input, selections):
     report = Report()
 
@@ -24,18 +26,18 @@ def generate_report(classof,type,risk_domain,sep,lifecycle):
         type = type,
         description = LangValue(
             lang = 'eng',
-            value = scraped_input['title']
+            value = title
         )
     )
     report.references = [
         Reference(
-            label = scraped_input['description'],
-            url = scraped_input['url']
+            label = title,
+            url = uri
         )
     ]
     report.description = LangValue(
         lang = 'eng',
-        value = scraped_input['description']
+        value = abstract
     )
     report.impact = Impact(
         avid = AvidTaxonomy(
@@ -49,24 +51,28 @@ def generate_report(classof,type,risk_domain,sep,lifecycle):
     
     return report.dict()
 
-scraped_input = {
-    "title": "### title",
-    "description": "description",
-    "url": "https://link.to.arxiv.paper"
-}
-
-# selections = {
-#     "classof": ClassEnum.llm,
-#     "type": TypeEnum.detection,
-#     "avid": {
-#         "risk_domain": ["Security"],
-#         "sep": [SepEnum.E0101],
-#         "lifecycle": [LifecycleEnum.L05]
-#     }
-# }
+# parses out title and abstract: works only on arxiv or acl
+def parse_uri(uri):
+    if not uri.startswith("https://arxiv.org") and not uri.startswith("https://aclanthology.org"):
+        raise gr.Error("Please supply an arxiv.org or aclanthology.org link!")
+
+    try:
+        response = urllib.request.urlopen(uri)
+        html = response.read()
+        doc = BeautifulSoup(html, features="lxml")
+    except:
+        raise gr.Error("Cannot fetch information. Please check the link!")
+
+    if uri.startswith("https://arxiv.org"):
+        title = doc.find("meta", property="og:title")["content"]
+        abstract = doc.find("meta", property="og:description")["content"].replace("\n", "")
+    else:
+        title = doc.find("meta", property="og:title")["content"]
+        abstract = doc.find("div", {"class": "card-body"}).text[8:].replace("\n", "")
+
+    return f"### {title}\n", abstract
 
 demo = gr.Blocks(theme=gr.themes.Soft())
-# demo = gr.Blocks(theme='gradio/darkdefault')
 
 with demo:
 
@@ -88,9 +94,10 @@ with demo:
             ## Step 1: \n\
             Select a model and a method of detection.
             """)
+            input_uri = gr.Text("Enter URL")
             with gr.Box():
-                title = gr.Markdown(scraped_input['title'])
-                description = gr.Markdown(scraped_input['description'])
+                title = gr.Markdown("### ")
+                description = gr.Markdown("")
 
         with gr.Column(scale=3):
             gr.Markdown("""## Step 2: \
@@ -112,9 +119,15 @@ with demo:
             report_button = gr.Button("Generate Report")
             report_json = gr.Json(label="AVID Report")
 
+    input_uri.change(
+        fn=parse_uri,
+        inputs=[input_uri],
+        outputs=[title,description]
+    )
+
     report_button.click(
         fn=generate_report,
-        inputs=[classof,type,risk_domain,sep,lifecycle],
+        inputs=[input_uri,title,description,classof,type,risk_domain,sep,lifecycle],
         outputs=[report_json]
     )
 

requirements.txt

@@ -1,4 +1,5 @@
 gradio==3.35.2
+bs4
 
 # for avidtools
 pydantic