Update crash handler capabilities and, making use of that, add temp alloc instrumentation to OOM crash hotspots (To help identify leaks/suboptimal procedures)

Author: Dutchman101

Client/core/CCrashDumpWriter.cpp

@@ -19,13 +19,15 @@
 #include <SharedUtil.Detours.h>
 #include <SharedUtil.Misc.h>
 #include <SharedUtil.Memory.h>
+#include "../Shared/sdk/CrashTelemetry.h"
 
 #include <algorithm>
 #include <array>
 #include <atomic>
 #include <chrono>
 #include <cctype>
 #include <cstdint>
+#include <cstring>
 #include <exception>
 #include <errno.h>
 #include <intrin.h>
@@ -40,6 +42,9 @@
 #include <string_view>
 #include <variant>
 #include <vector>
+#if defined(_MSC_VER)
+    #include <new.h>
+#endif
 
 #if defined(_MSC_VER)
     #include <corecrt.h>
@@ -90,6 +95,16 @@ inline std::atomic<DWORD> g_initializationPhase{INIT_PHASE_MINIMAL};
 
 using CrashHandlerResult = std::variant<std::monostate, std::string, DWORD, std::exception_ptr>;
 
+#if defined(__cplusplus)
+[[noreturn]] void __cdecl CppNewHandler() noexcept;
+#endif
+
+#if defined(_MSC_VER)
+static int __cdecl CppNewHandlerBridge(size_t size) noexcept;
+#else
+static void CppNewHandlerBridge() noexcept;
+#endif
+
 [[nodiscard]] BOOL BUGSUTIL_DLLINTERFACE __stdcall IsFatalException(DWORD exceptionCode) noexcept
 {
     switch (exceptionCode)
@@ -175,6 +190,11 @@ static void StoreBasicExceptionInfo(_EXCEPTION_POINTERS* pException) noexcept
         info.timestamp = std::chrono::system_clock::now();
         info.threadId = GetCurrentThreadId();
         info.processId = GetCurrentProcessId();
+        const auto telemetryNote = CrashTelemetry::BuildAllocationTelemetryNote();
+        if (!telemetryNote.empty())
+        {
+            info.additionalInfo = telemetryNote;
+        }
         g_lastExceptionInfo = info;
     }
     catch (...)
@@ -491,6 +511,12 @@ static void LogEnhancedExceptionInfo(_EXCEPTION_POINTERS* pException) noexcept
             case EXCEPTION_GUARD_PAGE:
                 info.exceptionType = "SEH:GuardPage";
                 break;
+            case CUSTOM_EXCEPTION_CODE_OOM:
+                info.exceptionType = "User:OutOfMemory";
+                break;
+            case CUSTOM_EXCEPTION_CODE_WATCHDOG_TIMEOUT:
+                info.exceptionType = "User:WatchdogTimeout";
+                break;
             default:
                 if (info.exceptionCode >= 0xC0000000 && info.exceptionCode <= 0xCFFFFFFF)
                 {
@@ -509,6 +535,15 @@ static void LogEnhancedExceptionInfo(_EXCEPTION_POINTERS* pException) noexcept
 
         info.exceptionDescription = GetExceptionCodeDescription(info.exceptionCode);
 
+        if (auto telemetryNote = CrashTelemetry::BuildAllocationTelemetryNote(); !telemetryNote.empty())
+        {
+            if (!info.additionalInfo.empty())
+            {
+                info.additionalInfo.push_back('\n');
+            }
+            info.additionalInfo += telemetryNote;
+        }
+
         g_lastExceptionInfo = info;
     }
     catch (...)
@@ -518,6 +553,15 @@ static void LogEnhancedExceptionInfo(_EXCEPTION_POINTERS* pException) noexcept
     }
 }
 
+static void CaptureAllocationTelemetry(_EXCEPTION_POINTERS* pException) noexcept
+{
+    if (pException == nullptr || pException->ExceptionRecord == nullptr)
+        return;
+
+    StoreBasicExceptionInfo(pException);
+    LogEnhancedExceptionInfo(pException);
+}
+
 static std::variant<DWORD, std::string> HandleExceptionModern(_EXCEPTION_POINTERS* pException) noexcept
 {
     if (pException == nullptr || pException->ExceptionRecord == nullptr)
@@ -621,7 +665,12 @@ static std::mutex                                          g_handlerStateMutex;
 static std::atomic<PFNCHFILTFN>                            g_pfnCrashCallback{nullptr};
 static std::atomic<LPTOP_LEVEL_EXCEPTION_FILTER>           g_pfnOrigFilt{nullptr};
 static std::atomic<std::terminate_handler>                 g_pfnOrigTerminate{nullptr};
+#if defined(_MSC_VER)
+using CrtNewHandler = int(__cdecl*)(size_t);
+static std::atomic<CrtNewHandler>                          g_pfnOrigNewHandler{nullptr};
+#else
 static std::atomic<std::new_handler>                       g_pfnOrigNewHandler{nullptr};
+#endif
 static std::atomic<decltype(&SetUnhandledExceptionFilter)> g_pfnKernelSetUnhandledExceptionFilter{nullptr};
 static decltype(&SetUnhandledExceptionFilter)              g_kernelSetUnhandledExceptionFilterTrampoline = nullptr;
 
@@ -633,10 +682,45 @@ static std::atomic<bool>                g_bInPureCallHandler{false};
 static std::atomic<bool>                g_bInTerminateHandler{false};
 static std::atomic<bool>                g_bInNewHandler{false};
 
+#if defined(_MSC_VER)
+static int __cdecl CppNewHandlerBridge(size_t size) noexcept
+{
+    const auto telemetry = CrashTelemetry::CaptureContext();
+    if ((!telemetry.hasData || telemetry.requestedSize == 0) && size > 0)
+    {
+        CrashTelemetry::SetAllocationContext(size, nullptr, "operator new", "std::new_handler");
+    }
+
+    if (auto previous = g_pfnOrigNewHandler.load(std::memory_order_acquire))
+    {
+        return previous(size);
+    }
+
+    CppNewHandler();
+    return 0;
+}
+#else
+static void CppNewHandlerBridge() noexcept
+{
+    const auto telemetry = CrashTelemetry::CaptureContext();
+    if (!telemetry.hasData)
+    {
+        CrashTelemetry::SetAllocationContext(0, nullptr, "operator new", "std::new_handler");
+    }
+
+    if (auto previous = g_pfnOrigNewHandler.load(std::memory_order_acquire))
+    {
+        previous();
+        return;
+    }
+
+    CppNewHandler();
+}
+#endif
+
 LONG __stdcall CrashHandlerExceptionFilter(EXCEPTION_POINTERS* pExPtrs);
 
 [[noreturn]] void __cdecl CppTerminateHandler() noexcept;
-[[noreturn]] void __cdecl CppNewHandler() noexcept;
 void __cdecl              AbortSignalHandler(int signal) noexcept;
 [[noreturn]] void __cdecl PureCallHandler() noexcept;
 
@@ -758,6 +842,8 @@ void __cdecl AbortSignalHandler([[maybe_unused]] int signal) noexcept
     exPtrs.ExceptionRecord = &exRecord;
     exPtrs.ContextRecord = &ctx;
 
+    CaptureAllocationTelemetry(&exPtrs);
+
     PFNCHFILTFN callback = g_pfnCrashCallback.load(std::memory_order_acquire);
 
     if (callback != nullptr)
@@ -796,13 +882,19 @@ void __cdecl AbortSignalHandler([[maybe_unused]] int signal) noexcept
     LogHandlerEvent(DEBUG_PREFIX_PURECALL.data(), "Pure virtual function call detected");
     SafeDebugOutput(DEBUG_SEPARATOR);
 
-    EXCEPTION_RECORD* pExRecord{nullptr};
-    CONTEXT* pCtx{nullptr};
+    EXCEPTION_RECORD*  pExRecord{nullptr};
+    CONTEXT*           pCtx{nullptr};
     EXCEPTION_POINTERS exPtrs{};
 
+    const bool haveContext = BuildExceptionContext(exPtrs, pExRecord, pCtx, EXCEPTION_NONCONTINUABLE_EXCEPTION);
+    if (haveContext)
+    {
+        CaptureAllocationTelemetry(&exPtrs);
+    }
+
     PFNCHFILTFN callback = g_pfnCrashCallback.load(std::memory_order_acquire);
 
-    if (callback != nullptr && BuildExceptionContext(exPtrs, pExRecord, pCtx, EXCEPTION_NONCONTINUABLE_EXCEPTION))
+    if (callback != nullptr && haveContext)
     {
         LogHandlerEvent(DEBUG_PREFIX_PURECALL.data(), "Calling crash handler callback");
 
@@ -850,7 +942,11 @@ class CleanUpCrashHandler
 
         if (auto newHandler = g_pfnOrigNewHandler.exchange(nullptr, std::memory_order_seq_cst); newHandler != nullptr)
         {
+    #if defined(_MSC_VER)
+            _set_new_handler(newHandler);
+    #else
             std::set_new_handler(newHandler);
+    #endif
         }
 
         if (auto abortHandler = g_pfnOrigAbortHandler.exchange(nullptr, std::memory_order_seq_cst); abortHandler != nullptr)
@@ -985,7 +1081,11 @@ static void InstallCppHandlers() noexcept
 
     if (g_pfnOrigNewHandler.load(std::memory_order_acquire) == nullptr)
     {
-        std::new_handler previous = std::set_new_handler(CppNewHandler);
+#if defined(_MSC_VER)
+        CrtNewHandler previous = _set_new_handler(CppNewHandlerBridge);
+#else
+        std::new_handler previous = std::set_new_handler(CppNewHandlerBridge);
+#endif
         g_pfnOrigNewHandler.store(previous, std::memory_order_release);
         SafeDebugOutput("CrashHandler: C++ new handler installed\n");
     }
@@ -1468,7 +1568,11 @@ static void UninstallCrashHandlers() noexcept
 
     if (auto newHandler = g_pfnOrigNewHandler.exchange(nullptr, std::memory_order_acq_rel); newHandler != nullptr)
     {
+#if defined(_MSC_VER)
+        _set_new_handler(newHandler);
+#else
         std::set_new_handler(newHandler);
+#endif
     }
 
     if (auto abortHandler = g_pfnOrigAbortHandler.exchange(nullptr, std::memory_order_acq_rel); abortHandler != nullptr)
@@ -1569,13 +1673,24 @@ static bool BuildExceptionContext(EXCEPTION_POINTERS& outExPtrs, EXCEPTION_RECOR
 
     SafeDebugOutput(DEBUG_SEPARATOR);
 
+    if (auto telemetryNote = CrashTelemetry::BuildAllocationTelemetryNote(); !telemetryNote.empty())
+    {
+        SafeDebugPrintPrefixed(DEBUG_PREFIX_CPP, "%s\n", telemetryNote.c_str());
+    }
+
     EXCEPTION_RECORD*  pExRecord = nullptr;
     CONTEXT*           pCtx = nullptr;
     EXCEPTION_POINTERS exPtrs{};
 
+    const bool haveContext = BuildExceptionContext(exPtrs, pExRecord, pCtx, CPP_EXCEPTION_CODE);
+    if (haveContext)
+    {
+        CaptureAllocationTelemetry(&exPtrs);
+    }
+
     PFNCHFILTFN callback = g_pfnCrashCallback.load(std::memory_order_acquire);
 
-    if (callback != nullptr && BuildExceptionContext(exPtrs, pExRecord, pCtx, CPP_EXCEPTION_CODE))
+    if (callback != nullptr && haveContext)
     {
         SafeDebugPrintPrefixed(DEBUG_PREFIX_CPP, "Calling crash handler callback\n");
 
@@ -1647,6 +1762,12 @@ static void ReportCurrentCppException() noexcept
     SafeDebugOutput("C++ NEW HANDLER: Memory allocation failed\n");
     SafeDebugOutput(DEBUG_SEPARATOR);
 
+    if (auto telemetryNote = CrashTelemetry::BuildAllocationTelemetryNote(); !telemetryNote.empty())
+    {
+        SafeDebugOutput(telemetryNote.c_str());
+        SafeDebugOutput("\n");
+    }
+
     std::terminate();
 }
 

Client/core/CrashHandler.cpp

@@ -11,6 +11,7 @@
 #include "StdInc.h"
 #include "CFileFormat.h"
 #include "CPixelsManager.h"
+#include <CrashTelemetry.h>
 
 ///////////////////////////////////////////////////////////////
 // Object creation
@@ -750,6 +751,22 @@ bool CPixelsManager::ChangePixelsFormat(const CPixels& oldPixels, CPixels& newPi
     if (oldFormat == EPixelsFormat::UNKNOWN || newFormat == EPixelsFormat::UNKNOWN)
         return false;
 
+    const uint sourceBytes = oldPixels.GetSize();
+    uint       width = 0;
+    uint       height = 0;
+    GetPixelsSize(oldPixels, width, height);
+
+    SString telemetryDetail;
+    telemetryDetail.Format("%s->%s q=%d bytes=%u dims=%ux%u",
+                           EnumToString(oldFormat).c_str(),
+                           EnumToString(newFormat).c_str(),
+                           uiQuality,
+                           sourceBytes,
+                           width,
+                           height);
+    // Tag conversions here so crashes show which pixel formats/dimensions were being processed.
+    CrashTelemetry::Scope conversionScope(sourceBytes, oldPixels.GetData(), "Pixels::ChangeFormat", telemetryDetail.c_str());
+
     if (oldFormat == newFormat)
     {
         // No change

Client/core/Graphics/CPixelsManager.cpp

@@ -4,6 +4,7 @@
 #define MTA_CLIENT
 #define SHARED_UTIL_WITH_FAST_HASH_MAP
 #include "SharedUtil.h"
+#include <CrashTelemetry.h>
 
 #include <string.h>
 #include <stdio.h>

Client/mods/deathmatch/StdInc.h

@@ -747,6 +747,13 @@ bool CClientEntity::CallEvent(const char* szName, const CLuaArguments& Arguments
     if (!g_pClientGame->GetDebugHookManager()->OnPreEvent(szName, Arguments, this, NULL))
         return false;
 
+    const char* thisTypeName = GetTypeName() ? GetTypeName() : "<unknown>";
+    const ElementID thisId = GetID();
+    SString telemetryDetail;
+    telemetryDetail.Format("%s %s(%u)", szName, thisTypeName, static_cast<unsigned int>(thisId));
+    // Capture the element+event context so any crash (even core.dll faults) reports the last event being dispatched.
+    CrashTelemetry::Scope entityScope(0, this, "Entity::CallEvent", telemetryDetail.c_str());
+
     TIMEUS startTime = GetTimeUs();
 
     CEvents* pEvents = g_pClientGame->GetEvents();

Client/mods/deathmatch/logic/CClientEntity.cpp

@@ -187,6 +187,30 @@ bool CMapEventManager::Call(const char* szName, const CLuaArguments& Arguments,
                     if (!g_pClientGame->GetDebugHookManager()->OnPreEventFunction(szName, Arguments, pSource, nullptr, pMapEvent))
                         continue;
 
+                    const char* scriptName = pMapEvent->GetVM()->GetScriptName();
+                    const CResource* pEventResource = pMapEvent->GetVM()->GetResource();
+                    const char* resourceName = pEventResource ? pEventResource->GetName().c_str() : "<no-resource>";
+
+                    SString sourceTag = "<null>";
+                    if (pSource)
+                    {
+                        const char* sourceTypeName = pSource->GetTypeName() ? pSource->GetTypeName() : "<unknown>";
+                        sourceTag.Format("%s(%u)", sourceTypeName, static_cast<unsigned int>(pSource->GetID()));
+                    }
+
+                    SString thisTag = "<null>";
+                    if (pThis)
+                    {
+                        const char* thisTypeName = pThis->GetTypeName() ? pThis->GetTypeName() : "<unknown>";
+                        thisTag.Format("%s(%u)", thisTypeName, static_cast<unsigned int>(pThis->GetID()));
+                    }
+
+                    SString telemetryDetail;
+                    telemetryDetail.Format("%s::%s src=%s this=%s", scriptName, resourceName, sourceTag.c_str(), thisTag.c_str());
+                    // Tag each Lua event dispatch so crash dumps show which
+                    // resource/event/element executed last
+                    CrashTelemetry::Scope eventScope(0, pThis, "LuaEvent::Call", telemetryDetail.c_str());
+
                     // Store the current values of the globals
                     lua_getglobal(pState, "source");
                     CLuaArgument OldSource(pState, -1);

Client/mods/deathmatch/logic/CMapEventManager.cpp

@@ -202,6 +202,9 @@ bool CLuaArguments::Call(CLuaMain* pLuaMain, const CLuaFunctionRef& iLuaFunction
     assert(pLuaMain);
     TIMEUS startTime = GetTimeUs();
 
+    const SString& functionTag = pLuaMain->GetFunctionTag(iLuaFunction.ToInt());
+    CrashTelemetry::Scope telemetryScope(0, pLuaMain, "Lua::Call", functionTag.c_str());
+
     // Add the function name to the stack and get the event from the table
     lua_State* luaVM = pLuaMain->GetVirtualMachine();
     assert(luaVM);
@@ -244,7 +247,7 @@ bool CLuaArguments::Call(CLuaMain* pLuaMain, const CLuaFunctionRef& iLuaFunction
             lua_pop(luaVM, 1);
     }
 
-    CClientPerfStatLuaTiming::GetSingleton()->UpdateLuaTiming(pLuaMain, pLuaMain->GetFunctionTag(iLuaFunction.ToInt()), GetTimeUs() - startTime);
+    CClientPerfStatLuaTiming::GetSingleton()->UpdateLuaTiming(pLuaMain, functionTag, GetTimeUs() - startTime);
     return true;
 }
 
@@ -254,6 +257,8 @@ bool CLuaArguments::CallGlobal(CLuaMain* pLuaMain, const char* szFunction, CLuaA
     assert(szFunction);
     TIMEUS startTime = GetTimeUs();
 
+    CrashTelemetry::Scope telemetryScope(0, pLuaMain, "Lua::CallGlobal", szFunction);
+
     // Add the function name to the stack and get the event from the table
     lua_State* luaVM = pLuaMain->GetVirtualMachine();
     assert(luaVM);