local logs_folder = "AppData/crashes" function GatherMinidumps(ignore_pattern) local err, files = AsyncListFiles(logs_folder, "*.dmp", "recursive,modified") if err then print(string.format("Crash folder enum error: %s", err)) return end if ignore_pattern then for i = #files, 1, -1 do local filepath = files[i] local _, filename = SplitPath(filepath) if string.match(filename, ignore_pattern) then table.remove(files, i) table.remove(files.modified, i) end end end return files end local function check(str, what) return string.starts_with(str, what, true) end function CrashFileParse(crash_file) local info = {} local crash_section_found, crash_section_complete local err, lines = AsyncFileToString(crash_file, nil, nil, "lines") if err then return err end PauseInfiniteLoopDetection("CrashFileParse") local crash_keys = { "Thread", "Module", "Address", "Function", "Process", "Error", "Details" } local header_keys = {"Lua revision", "Timestamp", "CPU", "GPU" } local patterns = { ["Lua revision"] = '^Lua revision:%s*(%d+)', ["Timestamp"] = "^Timestamp:%s*(%x+)", ["CPU"] = "^CPU%s*(.+)", ["GPU"] = "^GPU%s*(.+)", } local values = {} local _ local bR = string.byte("R") local b_ = string.byte("-") local bkeys, hkeys = {}, {} for i, key in ipairs(crash_keys) do bkeys[i] = string.byte(key) end for i, key in ipairs(header_keys) do hkeys[i] = string.byte(key) end for i, line in ipairs(lines) do local b = string.byte(line) for i, key in ipairs(header_keys) do if b == hkeys[i] and check(line, key) then local pattern = patterns[key] or ('^' .. key .. ':%s+(.+)$') local value = string.match(line, pattern) value = value and string.trim_spaces(value) if value then value = string.gsub(value, "[\n\r]", "") if key == "GPU" then local idx = string.find_lower(value, 'Feature Level') or string.find_lower(value, '{') if idx then value = string.sub(value, 1, idx - 1) value = string.trim_spaces(value) end elseif key == "CPU" then if string.starts_with(value, 'name', true) then value = string.sub(value, 5) value = string.trim_spaces(value) end end info[#info + 1] = key .. ": " .. value values[key] = value table.remove(header_keys, i) table.remove(hkeys, i) end break end end if crash_section_complete then -- elseif not crash_section_found then if b_ == b and check(line, "-- Exception Information") then crash_section_found = true end else if b == bR and check(line, "Registers:") or #crash_keys == 0 then crash_section_complete = true else for i, key in ipairs(crash_keys) do if b == bkeys[i] and check(line, key) then local value = string.match(line, '^' .. key .. ':%s+(.+)$') value = value and string.trim_spaces(value) if value then info[#info + 1] = key .. ": " .. value if key == "Thread" then _, value = string.match(value, '^(%d+)%s*\"(.+)\"$') elseif key == "Address" then value = string.sub(value, -4) end values[key] = value table.remove(crash_keys, i) table.remove(bkeys, i) end break end end end end if (#crash_keys == 0 or crash_section_complete) and (#header_keys == 0 or i > 1024) then break end end ResumeInfiniteLoopDetection("CrashFileParse") if not crash_section_found then return "Crash info not found" end local hash = xxhash(values.Address, values.Thread, values.Error, values.Details) local label = string.format("[Crash] @%s%s%s (%s) %s%s%s", values.Address or "", values.Function and " " or "", values.Function or "", values.Thread or "", values.Error or "", values.Details and ": " or "", values.Details or "") local revision = values["Lua revision"] local revision_num = revision and tonumber(revision) or 0 local info_str = table.concat(info, "\n") return nil, info_str, label, values, revision_num, hash end function CrashUploadToMantis(minidumps) local exception_info = {} local min_revision = config.BugReportCrashesMinRevision or 0 local unmount local function report(dump_file) local dump_dir, dump_name, dump_ext = SplitPath(dump_file) local crash_file = dump_dir .. dump_name .. ".crash" local err, info_str, label, values, revision_num, hash = CrashFileParse(crash_file) if err or not info_str or revision_num < min_revision or exception_info[hash] then return end exception_info[hash] = true if MountsByPath("memorytmp") == 0 then local err = MountPack("memorytmp", "", "create", 16*1024*1024) if err then print("MountPack error:", err) return end unmount = true end local pack_file = "memorytmp/" .. dump_name .. ".hpk" local pack_index = { { src = dump_file, dst = dump_name .. ".dmp", }, } local err, log = AsyncPack(pack_file, "", pack_index) if err then print("Pack error:", err) return end --print(os.date("%T"), ConvertToOSPath(crash_file)) local files = { crash_file, pack_file } local descr = "All crash and dump files are already attached." WaitXBugReportDlg(label, descr, files, { summary_readonly = true, no_screenshot = true, no_extra_info = true, append_description = "\n----\n" .. info_str, tags = { "Crash" }, severity = "crash", }) AsyncFileDelete(pack_file) end for _, minidump in ipairs(minidumps) do report(minidump) end if unmount then local err = UnmountByPath("memorytmp") if err then print("UnmountByPath error:", err) return end end end function MinidumpUploadAsync(url, os_path) local err, json = LuaToJSON({upload_file_minidump=os_path}) if err then print("Failed to convert minidump data to JSON", err) return end local err, info = AsyncWebRequest{ url = url, method = "POST", headers = {["Content-Type"] = "application/json"}, body = json, } err = err or info and info.error if err then print(string.format("Minidump upload fail: %s", err)) end end function GetCrashFiles(file_spec) local _, crash_files = AsyncListFiles(logs_folder, file_spec or "*.crash", "recursive,modified") crash_files = crash_files or {} local crash_date, index = table.max(crash_files.modified) return crash_files, crash_files[index], crash_date, index end function EmptyCrashFolder() return AsyncEmptyPath(logs_folder) end function CrashReportingEnabled() if not Platform.pc then return end return config.UploadMinidump or config.BugReportCrashesOnStartup end function RenameCrashPair(minidump, new_minidump) AsyncFileRename(minidump, new_minidump) local crash_file = string.gsub(minidump, ".dmp$", ".crash") local new_crash_file = string.gsub(new_minidump, ".dmp$", ".crash") AsyncFileRename(crash_file, new_crash_file) end if FirstLoad then g_bCrashReported = false end function WaitBugReportCrashesOnStartup() local _, minidump = GetCrashFiles("*.dmp") if not minidump then return end CrashUploadToMantis({minidump}) EmptyCrashFolder() end function OnMsg.EngineStarted() if not config.BugReportCrashesOnStartup or g_bCrashReported then return end g_bCrashReported = true CreateRealTimeThread(WaitBugReportCrashesOnStartup) end ---- if FirstLoad then SymbolsFolders = false GedFolderCrashesInstance = false CrashCache = false CrashFilter = false CrashResolved = false end CrashCacheVersion = 4 local base_cache_folder = "AppData/CrashCache/" local cache_file = base_cache_folder .. "CrashCache.bin" local resolved_file = ConvertToBenderProjectPath("Logs/Crashes/__Resolved.lua") CrashFolderSymbols = ConvertToBenderProjectPath("Logs/Pdbs/") CrashFolderBender = ConvertToBenderProjectPath("Logs/Crashes") CrashFolderSwarm = ConvertToBenderProjectPath("SwarmBackup/*/Storage/log-crash") CrashFolderLocal = "AppData/crashes" local defaults_groups = { SwarmBackup = ">Swarm", } local CrashInfoButtons = { {name = "LocateSymbols", func = "SymbolsFolderOpen"}, } DefineClass.CrashInfo = { __parents = {"PropertyObject"}, properties = { { category = "Actions", id = "Actions", editor = "buttons", default = "", buttons = CrashInfoButtons }, { category = "Crash", id = "ExeTimestamp", name = "Exe Timestamp", editor = "text", default = "" }, { category = "Crash", id = "SymbolsFolder", name = "Symbols Folder", editor = "text", default = "", buttons = {{name = "Open", func = "SymbolsFolderOpen"}} }, } } function CrashInfo:SymbolsFolderOpen() local bdb_folder = self.SymbolsFolder if bdb_folder ~= 0 then local os_command = string.format("cmd /c start \"\" \"%s\"", bdb_folder) os.execute(os_command) end end function CrashInfo:GetCacheFolder() local timestamp = self.ExeTimestamp if timestamp == "" then return "Missing timestamp!" end local cache_folder = base_cache_folder .. timestamp .. "/" if not io.exists(cache_folder) then local err = AsyncCreatePath(cache_folder) if err then return err end end return nil, cache_folder end function GedFolderCrashesRun(get) local crash = get.selected_object if crash then crash:OpenLogFile() end end DefineClass.FolderCrashGroup = { __parents = {"SortedBy", "GedFilter" }, properties = { { id = "name", editor = "text", default = "", read_only = true, buttons = {{name = "Export", func = "ExportToCSV"}} }, { id = "count", editor = "number", default = 0, read_only = true }, { id = "thread", name = "Show Thread", editor = "combo", default = false, items = function(self) return table.keys(self.threads, true) end }, { id = "timestamp", name = "Show Timestamp", editor = "combo", default = false, items = function(self) return table.keys(self.timestamps, true) end }, { id = "filter", name = "Show Name", editor = "combo", default = false, items = function(self) return table.keys(self.names, true) end }, { id = "cpu", name = "Show CPU", editor = "combo", default = false, items = function(self) return table.keys(self.cpus, true) end }, { id = "gpu", name = "Show GPU", editor = "combo", default = false, items = function(self) return table.keys(self.gpus, true) end }, { id = "unique", name = "Show Unique Only", editor = "bool", default = false }, { id = "resolved", name = "Show Resolved", editor = "bool", default = false }, { id = "shown_count", name = "Shown Count", editor = "number", default = 0, read_only = true }, }, shown = false, names = false, timestamps = false, threads = false, cpus = false, gpus = false, } function FolderCrashGroup:PrepareForFiltering() self.shown = {} self.shown_count = 0 end function FolderCrashGroup:FilterObject(obj) local name = obj.name if self.unique then if self.shown[name] then return end self.shown[name] = true end if not self.resolved and CrashResolved and CrashResolved[obj.hash] then return end local timestamp = self.timestamp if timestamp and timestamp ~= obj.ExeTimestamp then return end local thread = self.thread if thread and thread ~= obj.thread then return end local cpu = self.cpu if cpu and cpu ~= obj.CPU then return end local gpu = self.gpu if gpu and gpu ~= obj.GPU then return end local filter = self.filter if filter and filter ~= name and not string.find(name, filter) then return end self.shown_count = self.shown_count + 1 return true end function FolderCrashGroup:ExportToCSV() local name = string.starts_with(self.name, ">") and string.sub(self.name, 2) or self.name local path = base_cache_folder .. name .. ".csv" local err = SaveCSV(path, self, {"name", "thread", "date", "CPU", "GPU", "ExeTimestamp"}, {"name", "thread", "date", "CPU", "GPU", "Exe"}) if err then print(err, "while exporting", path) else print("Exported to", path) OpenTextFileWithEditorOfChoice(path) end end function FolderCrashGroup:GetSortItems() return {"name", "timestamp", "thread", "date", "CPU", "GPU", "occurrences"} end function FolderCrashGroup:Cmp(c1, c2, sort_by) local n1, n2 = c1.name, c2.name local ts1, ts2 = c1.ExeTimestamp, c2.ExeTimestamp local d1, d2 = c1.DmpTimestamp, c2.DmpTimestamp local CPU1, CPU2 = c1.CPU, c2.CPU local GPU1, GPU2 = c1.GPU, c2.GPU local o1, o2 = c1.occurrences, c2.occurrences if sort_by == "occurrences" then if o1 ~= o2 then return o1 > o2 end elseif sort_by == "date" then if d1 ~= d2 then return d1 < d2 end elseif sort_by == "thread" then local t1, t2 = c1.thread, c2.thread if t1 ~= t2 then return t1 < t2 end elseif sort_by == "timestamp" then if ts1 ~= ts2 then return ts1 < ts2 end elseif sort_by == "CPU" then if CPU1 ~= CPU2 then return CPU1 < CPU2 end elseif sort_by == "GPU" then if GPU1 ~= GPU2 then return GPU1 < GPU2 end end if n1 ~= n2 then return n1 < n2 end if ts1 ~= ts2 then return ts1 < ts2 end if d1 ~= d2 then return d1 < d2 end if o1 ~= o2 then return o1 > o2 end if GPU1 ~= GPU2 then return GPU1 < GPU2 end if CPU1 ~= CPU2 then return CPU1 < CPU2 end end function FolderCrashGroup:GetEditorView() return string.format("%s %d", self.name, self.count) end local FolderCrashButtons = { {name = "DebugInVS", func = "DebugDump"}, {name = "LocateSymbols", func = "SymbolsFolderOpen"}, {name = "OpenLog", func = "OpenLogFile"}, {name = "Resolve", func = "ResolveCrash"}, } DefineClass.FolderCrash = { __parents = {"CrashInfo"}, properties = { { category = "Actions", id = "Actions", editor = "buttons", default = "", buttons = FolderCrashButtons }, { category = "Actions", id = "Resolved", editor = "bool", default = false, read_only = true }, { category = "Crash", id = "ModuleName", editor = "text", default = "" }, { category = "Crash", id = "LocalModuleName", editor = "text", default = "", help = "Use it to change the symbols name locally, if the expected PDB name do not match" }, { category = "Crash", id = "name", name = "Summary", editor = "text", default = "" }, { category = "Crash", id = "occurrences", name = "Occurrences", editor = "number", default = 0, }, { category = "Crash", id = "date", name = "Dmp Date", editor = "text", default = "" }, { category = "Crash", id = "DmpTimestamp", name = "Dmp Timestamp", editor = "number", default = 0 }, { category = "Crash", id = "thread", editor = "text", default = "" }, { category = "Crash", id = "CPU", editor = "text", default = "" }, { category = "Crash", id = "GPU", editor = "text", default = "" }, { category = "Crash", id = "full_path", name = "Log Path", editor = "text", default = "", buttons = {{name = "Open", func = "OpenLogFile"}}, }, { category = "Crash", id = "crash_info", name = "Full Info", editor = "text", default = "", max_lines = 30, lines = 10, }, { category = "Crash", id = "dump_file", name = "text", editor = "text", default = "", no_edit = true, }, { category = "Crash", id = "group", name = "text", editor = "text", default = "", no_edit = true, }, { category = "Crash", id = "values", editor = "prop_table", default = false, no_edit = true, }, { category = "Crash", id = "hash", editor = "number", default = false, no_edit = true, }, }, StoreAsTable = true, CustomModuleName = false, } function WaitSaveCrashResolved() local code = pstr("return ", 1024) TableToLuaCode(CrashResolved, nil, code) local err = AsyncStringToFile(resolved_file, code) if err then print("once", "Failed to save the resolved crashes to", resolved_file, ":", err) end end function FolderCrash:GetModuleNameRaw() local module_file = self.values and self.values.Module if (module_file or "") == "" then return "" end local module_dir, module_name, module_ext = SplitPath(module_file) return module_name end function FolderCrash:GetModuleName() if (self.CustomModuleName or "") ~= "" then return self.CustomModuleName end return self:GetModuleNameRaw() end function FolderCrash:SetModuleName(module_name) self.CustomModuleName = (module_name or "") ~= "" and module_name ~= self:GetModuleNameRaw() and module_name or nil end function FolderCrash:GetResolved() return CrashResolved and CrashResolved[self.hash] end function FolderCrash:ResolveCrash(root, prop_id, ged) if self:GetResolved() then print(self.name, "is already resolved") return end if ged:WaitQuestion("Resolve", string.format("Mark crash \"%s\" as resolved?", self.name), "Yes", "No") ~= "ok" then return end CrashResolved = CrashResolved or {} CrashResolved[self.hash] = self.name .. " " .. self.ExeTimestamp DelayedCall(0, WaitSaveCrashResolved) end function FolderCrash:GetEditorView() local resolved = self:GetResolved() local color_start = resolved and "RESOLVED " or "" local color_end = resolved and "" or "" return string.format("", color_start, self.name, color_end, self.ExeTimestamp, self.CPU, self.GPU) end function FolderCrash:OpenLogFile() local full_path = self.full_path or "" if full_path ~= "" then OpenTextFileWithEditorOfChoice(full_path) end end function CopySymbols(cache_folder, src_folder, module_name, local_name) if (module_name or "") == "" then return "Invalid param!" end if (local_name or "") == "" then local_name = module_name end local pdbfile = cache_folder .. local_name .. ".pdb" if io.exists(pdbfile) then print("Using locally cached", pdbfile) return end if src_folder == "" then return "Symbols folder not found!" end local err, files = AsyncListFiles(src_folder, module_name .. ".*") if err then return print_format("Failed to list", src_folder, ":", err) end for _, file in ipairs(files) do local file_dir, file_name, file_ext = SplitPath(file) local dest = cache_folder .. local_name .. file_ext print("Copying", file, "to", dest) local err = AsyncCopyFile(file, dest, "raw") if err then return print_format("Failed to copy", file, ":", err) end end if not io.exists(pdbfile) then return print_format("No symbols found at", src_folder) end end function FolderCrash:DebugDump() if not Platform.pc then print("Supported on PC only!") return end local err local module_name = self:GetModuleName() or "" if module_name == "" or string.lower(module_name) == "unknown" then print("Invalid module name!") return end local err, cache_folder = self:GetCacheFolder() if err then print("Failed to create working directory:", err) return end local orig_dump_file = self.dump_file local orig_dump_dir, dump_name, dump_ext = SplitPath(orig_dump_file) local dump_file = cache_folder .. dump_name .. dump_ext if not io.exists(dump_file) then if not io.exists(orig_dump_file) then print("No dump pack found!") return end local err = AsyncCopyFile(orig_dump_file, dump_file, "raw") if err then print("Failed to copy", orig_dump_file, ":", err) return end end local err = CopySymbols(cache_folder, self.SymbolsFolder, module_name, self.LocalModuleName) if err then print("Copy symbols error:", err) return end local os_path = ConvertToOSPath(dump_file) local os_command = string.format("cmd /c start \"\" \"%s\"", os_path) os.execute(os_command) end function FetchSymbolsFolders() local err local st = GetPreciseTicks() err, SymbolsFolders = AsyncListFiles(CrashFolderSymbols, "*", "folders") if err then print("Failed to fetch symbols folders from Bender:", err) SymbolsFolders = {} end print(#SymbolsFolders, "symbol folders found in", GetPreciseTicks() - st, "ms at", CrashFolderSymbols) end function ResolveSymbolsFolder(timestamp) if (timestamp or "") == "" then return end assert(SymbolsFolders) for _, folder in ipairs(SymbolsFolders) do if string.ends_with(folder, timestamp, true) then return folder end end end function OpenCrashFolderBrowser(location, timestamp) CreateRealTimeThread(WaitOpenCrashFolderBrowser, location, timestamp) end function WaitOpenCrashFolderBrowser(location, timestamp) print("Opening crash folder browser at", location) FetchSymbolsFolders() if not CrashCache then local err, str = AsyncFileToString(cache_file) if not err then CrashCache = dostring(str) end if not CrashCache or CrashCache.version ~= CrashCacheVersion then CrashCache = { version = CrashCacheVersion } end end if not CrashResolved then local err, str = AsyncFileToString(resolved_file) if not err then CrashResolved = dostring(str) end if not CrashResolved then CrashResolved = {} end end local to_read, to_delete = {}, {} local to_delete_count = 0 local groups = {} local total_count = 0 local function AddCrashTo(crash, crash_name, group_name) local group = groups[group_name] if not group then group = FolderCrashGroup:new{ name = group_name } groups[group_name] = group groups[#groups + 1] = group end group[#group + 1] = crash end local skipped = 0 local function AddCrash(crash, group_name) if timestamp and timestamp ~= crash.ExeTimestamp then skipped = skipped + 1 return end AddCrashTo(crash, crash.name, crash.group) AddCrashTo(crash, crash.name, ">All") total_count = total_count + 1 end local created = 0 local read = 0 local function ReadCrash(info) read = read + 1 local crashfile, folder = info[1], info[2] local file_dir, file_name, file_ext = SplitPath(crashfile) local dump_file = file_dir .. file_name .. ".dmp" local err, info, label, values, revision_num, hash, DmpTimestamp err, DmpTimestamp = AsyncGetFileAttribute(dump_file, "timestamp") if err then print(err, "while getting timestamp of", dump_file) else err, info, label, values, revision_num, hash = CrashFileParse(crashfile) if err then print(err, "error while reading", crashfile) end end if err then to_delete_count = to_delete_count + 1 to_delete[#to_delete + 1] = crashfile to_delete[#to_delete + 1] = dump_file return end local group_name = string.sub(file_dir, #folder + 2) if group_name == "" then group_name = ">Ungrouped" for pattern, name in pairs(defaults_groups) do if file_dir:find(pattern) then group_name = name break end end else group_name = group_name:sub(1, -2) group_name = group_name:gsub("\\", "/") end local crash = FolderCrash:new{ dump_file = dump_file, group = group_name, folder = file_dir, name = label, full_path = crashfile, crash_info = info, date = os.date("%y/%m/%d %H:%M:%S", DmpTimestamp), DmpTimestamp = DmpTimestamp, ExeTimestamp = values.Timestamp, SymbolsFolder = ResolveSymbolsFolder(values.Timestamp), CPU = values.CPU, GPU = values.GPU, thread = values.Thread, values = values, hash = hash, } CrashCache[crashfile] = crash AddCrash(crash) created = created + 1 if read % 100 == 0 then print(#to_read - read, "remaining...") end end local folders if type(location) == "string" then folders = { location } elseif type(location) == "table" then folders = location else folders = { CrashFolderBender } end print("Fetching folder structure...") while true do local found for i=#folders,1,-1 do local folder = folders[i] local star_i = folder:find_lower("*") if star_i then found = true table.remove(folders, i) local base = folder:sub(1, star_i - 1) local sub = folder:sub(star_i + 1) local err, subfolders = AsyncListFiles(base, "*", "folders") if err then print("Failed to fetch issues from", base, ":", err) else for _, subfolder in ipairs(subfolders) do local f1 = subfolder .. sub if io.exists(f1) then folders[#folders + 1] = f1 end end end end end if not found then break end end for _, folder in ipairs(folders) do if folder:ends_with("/") or folder:ends_with("\\") then folder = folder:sub(1, -2) end local st = GetPreciseTicks() local err, files = AsyncListFiles(folder, "*.crash", "recursive") if err then printf("Failed to fetch issues (%s) from '%s'", err, folder) else printf("%d crashes found in '%s'", #files, folder) for i, crashfile in ipairs(files) do local group_name local cache = CrashCache[crashfile] if cache then AddCrash(cache) else to_read[#to_read + 1] = { crashfile, folder } end end end end local st = GetPreciseTicks() parallel_foreach(to_read, ReadCrash) table.sortby_field(groups, "name") for _, group in ipairs(groups) do local names, timestamps, threads, gpus, cpus = {}, {}, {}, {}, {} group.names = names group.timestamps = timestamps group.threads = threads group.gpus = gpus group.cpus = cpus for _, crash in ipairs(group) do local name = crash.name names[name] = (names[name] or 0) + 1 timestamps[crash.ExeTimestamp] = true threads[crash.thread] = true gpus[crash.GPU] = true cpus[crash.CPU] = true end for _, crash in ipairs(group) do crash.occurrences = names[crash.name] end group:Sort() group.count = #group end print("Crashes processed:", total_count, ", skipped:", skipped, ", time:", GetPreciseTicks() - st, "ms") if created > 0 then local code = pstr("return ", 1024) TableToLuaCode(CrashCache, nil, code) AsyncCreatePath(base_cache_folder) local err = AsyncStringToFile(cache_file, code, -2, 0, "zstd") if err then print("once", "Failed to save the crash cache to", cache_file, ":", err) end end local ged = OpenGedAppSingleton("GedFolderCrashes", groups) ged:SetSelection("root", { 1 }, nil, not "notify") if to_delete_count > 0 then if "ok" == WaitQuestion(terminal.desktop, "Warning", string.format("Confirm removal of %s invalid crash files?", to_delete_count)) then local err = AsyncFileDelete(to_delete) if err then print(err, "while deleting invalid crash files!") else print(to_delete_count, "invalid crash files removed.") end end end end