Ali2206 commited on
Commit
ba04857
·
verified ·
1 Parent(s): 4951984

Update ui/ui_core.py

Browse files
Files changed (1) hide show
  1. ui/ui_core.py +2 -1
ui/ui_core.py CHANGED
@@ -12,7 +12,8 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..",
12
  from txagent.txagent import TxAgent
13
 
14
  def sanitize_utf8(text: str) -> str:
15
- return re.sub(r'[\ud800-\udfff]', '', text)
 
16
 
17
  def extract_all_text_from_csv_or_excel(file_path: str, progress=None, index=0, total=1) -> str:
18
  try:
 
12
  from txagent.txagent import TxAgent
13
 
14
  def sanitize_utf8(text: str) -> str:
15
+ clean_text = re.sub(r'[\ud800-\udfff]', '', text)
16
+ return clean_text.encode("utf-8", "ignore").decode("utf-8", "ignore")
17
 
18
  def extract_all_text_from_csv_or_excel(file_path: str, progress=None, index=0, total=1) -> str:
19
  try: