Update ui/ui_core.py
Browse files- ui/ui_core.py +2 -1
ui/ui_core.py
CHANGED
@@ -12,7 +12,8 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..",
|
|
12 |
from txagent.txagent import TxAgent
|
13 |
|
14 |
def sanitize_utf8(text: str) -> str:
|
15 |
-
|
|
|
16 |
|
17 |
def extract_all_text_from_csv_or_excel(file_path: str, progress=None, index=0, total=1) -> str:
|
18 |
try:
|
|
|
12 |
from txagent.txagent import TxAgent
|
13 |
|
14 |
def sanitize_utf8(text: str) -> str:
|
15 |
+
clean_text = re.sub(r'[\ud800-\udfff]', '', text)
|
16 |
+
return clean_text.encode("utf-8", "ignore").decode("utf-8", "ignore")
|
17 |
|
18 |
def extract_all_text_from_csv_or_excel(file_path: str, progress=None, index=0, total=1) -> str:
|
19 |
try:
|