cleaning up local files after write

utils/logger.py

@@ -96,6 +96,24 @@ class ChatLogger:
         
         return log_entry
 
+    def cleanup_local_files(self):
+        """Delete local JSON files after successful upload"""
+        try:
+            # List all files in json_dataset directory
+            for file in self.json_dataset_dir.glob("*.json*"):
+                try:
+                    file.unlink()  # Delete file
+                    logging.info(f"Deleted local file: {file}")
+                except Exception as e:
+                    logging.error(f"Error deleting file {file}: {e}")
+            
+            # Optionally remove the directory if empty
+            if not any(self.json_dataset_dir.iterdir()):
+                self.json_dataset_dir.rmdir()
+                logging.info("Removed empty json_dataset directory")
+        except Exception as e:
+            logging.error(f"Error in cleanup: {e}")
+
     def save_local(self, log_entry):
         """Save log entry to local JSONL file"""
         try:
@@ -117,6 +135,9 @@ class ChatLogger:
                     json.dump(ordered_logs, f)
                     f.write('\n')
                     logging.info("Log entry saved")
+                
+                # After successful write, trigger cleanup
+                self.cleanup_local_files()
             return True
         except Exception as e:
             logging.error(f"Error saving to local file: {str(e)}")