///////////////////////////////////////////////////////////////////////////////// // // Copyright (C) 2022 Equinor ASA // // ResInsight is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY // WARRANTY; without even the implied warranty of MERCHANTABILITY or // FITNESS FOR A PARTICULAR PURPOSE. // // See the GNU General Public License at // for more details. // ///////////////////////////////////////////////////////////////////////////////// #include "RiaMainTools.h" #include "RiaBaseDefs.h" #include "RiaFileLogger.h" #include "RiaGuiApplication.h" #include "RiaLogging.h" #include "RiaOpenTelemetryManager.h" #include "RiaPreferencesOpenTelemetry.h" #include "RiaRegressionTestRunner.h" #include "RiaSocketCommand.h" #include "RiaVersionInfo.h" #include "cafCmdFeature.h" #include "cafCmdFeatureManager.h" #include "cafPdmDefaultObjectFactory.h" #include "cafPdmUiFieldEditorHandle.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // std::stacktrace is C++23 and not usable on all toolchains. The build system // probes it by linking and may force it off via RIA_HAS_STD_STACKTRACE=0, see // cmake/StdStacktrace.cmake and #14045. #ifndef RIA_HAS_STD_STACKTRACE #if defined( __cpp_lib_stacktrace ) && __cpp_lib_stacktrace >= 202011L #define RIA_HAS_STD_STACKTRACE 1 #else #define RIA_HAS_STD_STACKTRACE 0 #endif #endif #if RIA_HAS_STD_STACKTRACE #include #endif #ifndef WIN32 #include #include // macOS marks as deprecated and the mcontext layout // (gregs[REG_RIP]) is Linux-only — skip the program-counter capture // path on Apple platforms. See #14045. #ifndef __APPLE__ #include #endif #endif #ifdef WIN32 #ifndef NOMINMAX #define NOMINMAX #endif #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include #include #endif #ifdef WIN32 //-------------------------------------------------------------------------------------------------- /// In headless mode Qt creates its OpenGL context using the software rasterizer opengl32sw.dll /// (Qt::AA_UseSoftwareOpenGL), while the statically linked OpenGL calls in the visualization /// framework would go to the system opengl32.dll where no context is current, causing an endless /// loop in glGetError(). opengl32.dll is therefore delay-loaded (see CMakeLists.txt) and redirected /// here to the same software rasterizer. //-------------------------------------------------------------------------------------------------- FARPROC WINAPI riaDelayLoadHook( unsigned dliNotify, PDelayLoadInfo pdli ) { if ( dliNotify == dliNotePreLoadLibrary && RiaGuiApplication::isHeadless() && _stricmp( pdli->szDll, "opengl32.dll" ) == 0 ) { if ( HMODULE softwareOpenGl = LoadLibraryW( L"opengl32sw.dll" ) ) { return reinterpret_cast( softwareOpenGl ); } } return nullptr; } extern "C" const PfnDliHook __pfnDliNotifyHook2 = riaDelayLoadHook; //-------------------------------------------------------------------------------------------------- /// ResInsight is built as a GUI application on Windows, so stdout is only available when redirected to a file or /// pipe. Attach to the console of the launching process (e.g. PowerShell or cmd) so headless log output is visible. //-------------------------------------------------------------------------------------------------- void riaAttachToParentConsole() { // Only files and pipes are real redirections. A GUI process launched from a console (e.g. Windows Terminal) can // inherit the console handles of the parent without being attached to the console, and writing to those blocks auto isRedirected = []( DWORD stdHandle ) { HANDLE handle = GetStdHandle( stdHandle ); if ( handle == nullptr || handle == INVALID_HANDLE_VALUE ) return false; const DWORD fileType = GetFileType( handle ); return fileType == FILE_TYPE_DISK || fileType == FILE_TYPE_PIPE; }; const bool stdOutRedirected = isRedirected( STD_OUTPUT_HANDLE ); const bool stdErrRedirected = isRedirected( STD_ERROR_HANDLE ); if ( stdOutRedirected && stdErrRedirected ) return; if ( !AttachConsole( ATTACH_PARENT_PROCESS ) ) return; FILE* stream = nullptr; if ( !stdOutRedirected ) freopen_s( &stream, "CONOUT$", "w", stdout ); if ( !stdErrRedirected ) freopen_s( &stream, "CONOUT$", "w", stderr ); std::cout.clear(); std::cerr.clear(); } #endif namespace internal { #if RIA_HAS_STD_STACKTRACE // Custom formatter for stacktrace std::string formatStacktrace( const std::stacktrace& st ) { std::stringstream ss; int frame = 0; for ( const auto& entry : st ) { ss << " [" << frame++ << "] " << entry.description() << " at " << entry.source_file() << ":" << entry.source_line() << "\n"; } return ss.str(); } #endif #ifndef WIN32 static std::string signalCodeDescription( int signo, int siCode ) { switch ( signo ) { case SIGSEGV: switch ( siCode ) { case SEGV_MAPERR: return "address not mapped (SEGV_MAPERR)"; case SEGV_ACCERR: return "invalid access permissions (SEGV_ACCERR)"; default: return "unknown SIGSEGV code"; } case SIGFPE: switch ( siCode ) { case FPE_INTDIV: return "integer divide by zero (FPE_INTDIV)"; case FPE_INTOVF: return "integer overflow (FPE_INTOVF)"; case FPE_FLTDIV: return "floating-point divide by zero (FPE_FLTDIV)"; case FPE_FLTOVF: return "floating-point overflow (FPE_FLTOVF)"; case FPE_FLTUND: return "floating-point underflow (FPE_FLTUND)"; case FPE_FLTRES: return "floating-point inexact result (FPE_FLTRES)"; case FPE_FLTINV: return "invalid floating-point operation (FPE_FLTINV)"; case FPE_FLTSUB: return "subscript out of range (FPE_FLTSUB)"; default: return "unknown SIGFPE code"; } case SIGILL: switch ( siCode ) { case ILL_ILLOPC: return "illegal opcode (ILL_ILLOPC)"; case ILL_ILLOPN: return "illegal operand (ILL_ILLOPN)"; case ILL_ILLADR: return "illegal addressing mode (ILL_ILLADR)"; case ILL_ILLTRP: return "illegal trap (ILL_ILLTRP)"; case ILL_PRVOPC: return "privileged opcode (ILL_PRVOPC)"; case ILL_PRVREG: return "privileged register (ILL_PRVREG)"; case ILL_COPROC: return "coprocessor error (ILL_COPROC)"; case ILL_BADSTK: return "internal stack error (ILL_BADSTK)"; default: return "unknown SIGILL code"; } case SIGBUS: switch ( siCode ) { case BUS_ADRALN: return "invalid address alignment (BUS_ADRALN)"; case BUS_ADRERR: return "nonexistent physical address (BUS_ADRERR)"; case BUS_OBJERR: return "object-specific hardware error (BUS_OBJERR)"; default: return "unknown SIGBUS code"; } default: return ""; } } #endif } // namespace internal static std::atomic s_crashHandlingInProgress( false ); static std::atomic s_crashHandlingThread; static constexpr size_t s_maxRecordedQtMessages = 5; static constexpr int s_maxQtMessageLength = 300; static std::atomic_flag s_qtMessageLock = ATOMIC_FLAG_INIT; static std::deque s_recentQtMessages; //-------------------------------------------------------------------------------------------------- /// Crash logging is not async-signal-safe and may deadlock. Arm a watchdog so the process is /// killed by SIGALRM (default action) instead of lingering as a hung process using CPU. /// The timeout covers the telemetry flush (connection timeout) plus a margin for symbol resolution. //-------------------------------------------------------------------------------------------------- static void startCrashWatchdog() { #ifndef WIN32 unsigned int timeoutSeconds = 15; if ( auto* prefs = RiaPreferencesOpenTelemetry::current() ) { timeoutSeconds = std::max( timeoutSeconds, static_cast( prefs->connectionTimeoutMs() / 1000 + 10 ) ); } signal( SIGALRM, SIG_DFL ); alarm( timeoutSeconds ); #endif } //-------------------------------------------------------------------------------------------------- /// Must be the first thing done in every crash handler, before any allocation or formatting, so that /// the watchdog is armed and nested crashes are caught before they can recurse. /// /// A nested crash on the handling thread (e.g. a paint event during the telemetry flush) exits /// immediately. A crash on another thread is parked so the first crash report can complete; the /// watchdog still bounds how long that takes. See #15035. //-------------------------------------------------------------------------------------------------- static void enterCrashHandling() { if ( s_crashHandlingInProgress.exchange( true ) ) { if ( s_crashHandlingThread.load() != std::this_thread::get_id() ) { for ( ;; ) { std::this_thread::sleep_for( std::chrono::hours( 1 ) ); } } std::_Exit( 1 ); } s_crashHandlingThread = std::this_thread::get_id(); startCrashWatchdog(); } //-------------------------------------------------------------------------------------------------- /// Shared crash logging: writes to file logger and OpenTelemetry. /// extraAttrs may include platform-specific context like fault address and signal code description. /// /// Note: Executing logging functions from a signal handler is not async-signal-safe, but works as /// expected on Windows. Behavior on Linux is undefined, but will work in most cases. /// https://github.com/gabime/spdlog/issues/1607 //-------------------------------------------------------------------------------------------------- static void performCrashLogging( int signalCode, const std::map& crashAttrs ) { // Block timer-driven application logic (e.g. gRPC idle processing) for the rest of the process // lifetime: the telemetry flush below pumps the event loop, and a timer slot firing there could // close the project and destroy objects the crashed code still uses, causing a nested crash. // The flag is set by enterCrashHandling(), which every crash handler calls first. auto extraAttrs = crashAttrs; { // Spin lock with try-semantics: the crash may have happened while this thread was recording a message if ( !s_qtMessageLock.test_and_set( std::memory_order_acquire ) ) { if ( !s_recentQtMessages.empty() ) { std::string joined; for ( const auto& message : s_recentQtMessages ) { joined += message + "\n"; } extraAttrs["crash.last_qt_messages"] = joined; } s_qtMessageLock.clear( std::memory_order_release ); } } #if RIA_HAS_STD_STACKTRACE auto st = std::stacktrace::current(); #endif auto loggers = RiaLogging::loggerInstances(); for ( auto logger : loggers ) { if ( auto fileLogger = dynamic_cast( logger ) ) { auto versionText = QString( STRPRODUCTVER ); auto str = QString( "Crash (signal: %1) - ResInsight version %2" ).arg( signalCode ).arg( versionText ); fileLogger->error( str.toStdString().data() ); for ( const auto& [key, value] : extraAttrs ) { std::string line = key + ": " + value; fileLogger->error( line.data() ); } #if RIA_HAS_STD_STACKTRACE std::string message = "Stack trace:\n" + internal::formatStacktrace( st ); logger->error( message.data() ); #else logger->error( "Stack trace: not available (compiled without )" ); #endif fileLogger->flush(); } } auto& otelManager = RiaOpenTelemetryManager::instance(); if ( otelManager.isEnabled() ) { #if RIA_HAS_STD_STACKTRACE otelManager.reportCrash( signalCode, st, extraAttrs ); #endif } } //-------------------------------------------------------------------------------------------------- /// Terminate handler for uncaught C++ exceptions. Unlike the signal handler, this runs BEFORE the /// stack is unwound: when no handler is found, libstdc++ calls std::terminate from the throw site, /// so the stack trace captured here points at where the exception was originally thrown rather than /// at the crash-logging plumbing. Also recovers the exception type and message, which a bare SIGABRT /// trace cannot provide. Runs outside any signal handler, so logging here is safe (no async-signal /// constraints, unlike performCrashLogging when reached via the signal path). //-------------------------------------------------------------------------------------------------- void manageTerminate() { enterCrashHandling(); std::map extraAttrs; if ( auto ex = std::current_exception() ) { try { std::rethrow_exception( ex ); } catch ( const std::exception& e ) { extraAttrs["crash.exception_type"] = typeid( e ).name(); extraAttrs["crash.exception_what"] = e.what(); } catch ( ... ) { extraAttrs["crash.exception_type"] = "non-std exception"; } } // performCrashLogging captures std::stacktrace::current() itself; called here (no signal, no // prior unwind) that trace still includes the original throw site. performCrashLogging( SIGABRT, extraAttrs ); // Restore the default action so abort() produces a core dump and a SIGABRT exit status instead of // being caught by the crash handler, which would exit silently as a nested crash. signal( SIGABRT, SIG_DFL ); std::abort(); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- void manageSegFailure( int signalCode ) { enterCrashHandling(); performCrashLogging( signalCode, {} ); // _Exit skips atexit handlers and static destructors, which can crash again after a fault (#15035) std::_Exit( 1 ); } #ifndef WIN32 //-------------------------------------------------------------------------------------------------- /// SA_SIGINFO-compatible handler: captures fault address, signal code, and program counter /// for enhanced crash diagnostics on Linux. //-------------------------------------------------------------------------------------------------- void manageSegFailureSA( int signalCode, siginfo_t* info, void* ucontext ) { enterCrashHandling(); std::map extraAttrs; if ( info ) { std::ostringstream addrStr; addrStr << "0x" << std::hex << reinterpret_cast( info->si_addr ); extraAttrs["crash.fault_address"] = addrStr.str(); std::string codeDesc = internal::signalCodeDescription( signalCode, info->si_code ); if ( !codeDesc.empty() ) { extraAttrs["crash.signal_code"] = codeDesc; } } // Program-counter capture relies on the Linux ucontext layout // (gregs[REG_RIP] for x86_64, mcontext.pc for aarch64). macOS exposes // a different layout and marks as deprecated, so skip it // there; remaining crash attributes (signal code, fault address) are // still recorded. See #14045. #if !defined( __APPLE__ ) #if defined( __x86_64__ ) if ( ucontext ) { auto* ctx = static_cast( ucontext ); std::ostringstream pcStr; pcStr << "0x" << std::hex << static_cast( ctx->uc_mcontext.gregs[REG_RIP] ); extraAttrs["crash.program_counter"] = pcStr.str(); } #elif defined( __aarch64__ ) if ( ucontext ) { auto* ctx = static_cast( ucontext ); std::ostringstream pcStr; pcStr << "0x" << std::hex << ctx->uc_mcontext.pc; extraAttrs["crash.program_counter"] = pcStr.str(); } #endif #else (void)ucontext; #endif performCrashLogging( signalCode, extraAttrs ); // _Exit skips atexit handlers and static destructors, which can crash again after a fault (#15035) std::_Exit( 1 ); } #endif namespace RiaMainTools { //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- bool isCrashHandlingInProgress() { return s_crashHandlingInProgress; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- void recordQtMessage( QtMsgType type, const QString& msg ) { if ( type == QtDebugMsg || type == QtInfoMsg ) return; // Spin lock with try-semantics: this may be called from any thread, and must never block the message handler if ( s_qtMessageLock.test_and_set( std::memory_order_acquire ) ) return; auto text = msg.left( s_maxQtMessageLength ).toStdString(); // Skip repeats of the previous message, so a warning in a hot loop cannot flush out the earlier context if ( s_recentQtMessages.empty() || s_recentQtMessages.back() != text ) { s_recentQtMessages.push_back( std::move( text ) ); if ( s_recentQtMessages.size() > s_maxRecordedQtMessages ) s_recentQtMessages.pop_front(); } s_qtMessageLock.clear( std::memory_order_release ); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- void initializeSingletons() { caf::CmdFeatureManager::createSingleton(); RiaRegressionTestRunner::createSingleton(); caf::PdmDefaultObjectFactory::createSingleton(); } //-------------------------------------------------------------------------------------------------- /// This method is used to release memory allocated by static functions. This enables use of memory allocation tools /// after the application has closed down. //-------------------------------------------------------------------------------------------------- void releaseSingletonAndFactoryObjects() { caf::CmdFeatureManager::deleteSingleton(); RiaRegressionTestRunner::deleteSingleton(); caf::PdmDefaultObjectFactory::deleteSingleton(); { auto factory = caf::Factory::instance(); factory->deleteCreatorObjects(); } { auto factory = caf::Factory::instance(); factory->deleteCreatorObjects(); } { auto factory = caf::Factory::instance(); factory->deleteCreatorObjects(); } } //-------------------------------------------------------------------------------------------------- /// On Linux, application settings are stored in a text file in the users home folder. On Windows, these settings are /// stored in Registry. Users have reported stale lock files the configuration directory. In some cases, these lock /// files can prevent the application from starting. It appears that the application start, but no GUI is displayed. /// /// This method deletes stale lock files. /// /// https://github.com/OPM/ResInsight/issues/12205 //-------------------------------------------------------------------------------------------------- void deleteStaleSettingsLockFiles() { auto organizationName = QString( RI_COMPANY_NAME ); auto applicationName = QString( RI_APPLICATION_NAME ); auto lockFilePath = QDir::homePath() + "/.config/" + organizationName + "/" + applicationName + ".conf.lock"; auto isLockStale = []( const QString& lockFilePath ) -> bool { QFileInfo lockFileInfo( lockFilePath ); if ( !lockFileInfo.exists() ) return false; QDateTime currentTime = QDateTime::currentDateTime(); int thresholdSeconds = 2 * 60; // 2 minutes return lockFileInfo.lastModified().secsTo( currentTime ) > thresholdSeconds; }; if ( isLockStale( lockFilePath ) ) { QFile lockFile( lockFilePath ); QString logMessage; if ( lockFile.remove() ) { logMessage = QString( "Deleted stale lock file: %1" ).arg( lockFilePath ); } else { logMessage = QString( "Tried, but failed to delete stale lock file: %1" ).arg( lockFilePath ); } qDebug() << logMessage; } } //-------------------------------------------------------------------------------------------------- /// Configure software OpenGL rendering for headless mode, allowing 3D views to be rendered and /// grabbed without a GPU and without showing any windows. Must be called before the QApplication /// object is created. //-------------------------------------------------------------------------------------------------- void enableHeadlessRendering() { // Use software OpenGL rendering so 3D views can be rendered without a GPU. On Windows this // makes Qt load opengl32sw.dll (Mesa llvmpipe). Must be set before the QApplication exists. QApplication::setAttribute( Qt::AA_UseSoftwareOpenGL ); #ifndef WIN32 // On Linux the attribute has no effect, force Mesa software rendering instead qputenv( "LIBGL_ALWAYS_SOFTWARE", "1" ); #else riaAttachToParentConsole(); #endif RiaGuiApplication::enableHeadlessMode(); } } // namespace RiaMainTools