Spaces:

RyanSaleh
/

web-search-api

Sleeping

Hansimov commited on Jan 10, 2024

Commit

c7c538d

1 Parent(s): 3dda344

:zap: [Enhance] WebpageContentExtractor: Escape dash, and ignore

Files changed (1) hide show

networks/network_configs.py CHANGED Viewed

@@ -10,13 +10,14 @@ IGNORE_CLASSES = [
     "navbar",
     # 163.com
     "post_(top)|(side)|(recommends)|(crumb)|(statement)|(next)|(jubao)",
-    "ntes-.*nav",
-    "nav-bottom",
 ]
 IGNORE_HOSTS = [
     "weibo.com",
     "hymson.com",
 ]
 REQUESTS_HEADERS = {

     "navbar",
     # 163.com
     "post_(top)|(side)|(recommends)|(crumb)|(statement)|(next)|(jubao)",
+    "ntes\-.*nav",
+    "nav\-bottom",
 ]
 IGNORE_HOSTS = [
     "weibo.com",
     "hymson.com",
+    "yahoo.com",
 ]
 REQUESTS_HEADERS = {