Kevin Hu
commited on
Commit
·
20174e9
1
Parent(s):
e2220ea
add callback to entity extraction (#1843)
Browse files### What problem does this PR solve?
### Type of change
- [x] Refactoring
- [ ]
- graphrag/index.py +4 -2
graphrag/index.py
CHANGED
|
@@ -86,13 +86,15 @@ def build_knowlege_graph_chunks(tenant_id: str, chunks: List[str], callback, ent
|
|
| 86 |
for i in range(len(chunks)):
|
| 87 |
tkn_cnt = num_tokens_from_string(chunks[i])
|
| 88 |
if cnt+tkn_cnt >= left_token_count and texts:
|
| 89 |
-
|
|
|
|
| 90 |
texts = []
|
| 91 |
cnt = 0
|
| 92 |
texts.append(chunks[i])
|
| 93 |
cnt += tkn_cnt
|
| 94 |
if texts:
|
| 95 |
-
|
|
|
|
| 96 |
|
| 97 |
callback(0.5, "Extracting entities.")
|
| 98 |
graphs = []
|
|
|
|
| 86 |
for i in range(len(chunks)):
|
| 87 |
tkn_cnt = num_tokens_from_string(chunks[i])
|
| 88 |
if cnt+tkn_cnt >= left_token_count and texts:
|
| 89 |
+
for b in range(0, len(texts), 16):
|
| 90 |
+
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
|
| 91 |
texts = []
|
| 92 |
cnt = 0
|
| 93 |
texts.append(chunks[i])
|
| 94 |
cnt += tkn_cnt
|
| 95 |
if texts:
|
| 96 |
+
for b in range(0, len(texts), 16):
|
| 97 |
+
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
|
| 98 |
|
| 99 |
callback(0.5, "Extracting entities.")
|
| 100 |
graphs = []
|