Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -11,7 +11,7 @@ from sentence_transformers import SentenceTransformer
|
|
11 |
from transformers import T5Tokenizer, T5ForConditionalGeneration
|
12 |
import torch
|
13 |
|
14 |
-
def crawl(start_url: str, max_depth: int = 1, delay: float = 0.1)
|
15 |
visited = set()
|
16 |
results = []
|
17 |
queue = deque([(start_url, 0)])
|
@@ -45,7 +45,7 @@ def crawl(start_url: str, max_depth: int = 1, delay: float = 0.1) -> Tuple[List[
|
|
45 |
except Exception as e:
|
46 |
print(f"Error crawling {url}: {e}")
|
47 |
|
48 |
-
return results, crawled_urls
|
49 |
|
50 |
def chunk_text(text: str, max_chunk_size: int = 1000) -> List[str]:
|
51 |
chunks = []
|
|
|
11 |
from transformers import T5Tokenizer, T5ForConditionalGeneration
|
12 |
import torch
|
13 |
|
14 |
+
def crawl(start_url: str, max_depth: int = 1, delay: float = 0.1) :
|
15 |
visited = set()
|
16 |
results = []
|
17 |
queue = deque([(start_url, 0)])
|
|
|
45 |
except Exception as e:
|
46 |
print(f"Error crawling {url}: {e}")
|
47 |
|
48 |
+
return results[:10], crawled_urls[:10]
|
49 |
|
50 |
def chunk_text(text: str, max_chunk_size: int = 1000) -> List[str]:
|
51 |
chunks = []
|