Spaces:
Sleeping
Sleeping
Update preprocessing.py
Browse files- preprocessing.py +0 -22
preprocessing.py
CHANGED
@@ -15,25 +15,3 @@ def read_file(file_path):
|
|
15 |
else:
|
16 |
raise ValueError("Unsupported file format. Only .docx and .txt are allowed.")
|
17 |
|
18 |
-
s
|
19 |
-
|
20 |
-
def save_to_db(chunks, topics=None):
|
21 |
-
"""Save chunks to SQLite database."""
|
22 |
-
conn = sqlite3.connect('dataset.db')
|
23 |
-
cursor = conn.cursor()
|
24 |
-
|
25 |
-
# Create table if not exists
|
26 |
-
cursor.execute('''
|
27 |
-
CREATE TABLE IF NOT EXISTS documents (
|
28 |
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
29 |
-
text TEXT,
|
30 |
-
topics TEXT
|
31 |
-
)
|
32 |
-
''')
|
33 |
-
|
34 |
-
# Insert chunks into the database
|
35 |
-
for chunk in chunks:
|
36 |
-
cursor.execute('INSERT INTO documents (text, topics) VALUES (?, ?)', (chunk, topics))
|
37 |
-
|
38 |
-
conn.commit()
|
39 |
-
conn.close()
|
|
|
15 |
else:
|
16 |
raise ValueError("Unsupported file format. Only .docx and .txt are allowed.")
|
17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|