[cDAC] Implement SetJITCompilerFlags for cDAC (#126595)

##  Summary
Implement `SetJITCompilerFlags` on `IXCLRDataModule2` in the cDAC.

##  Changes
- ClrDataModule.cs - `SetJITCompilerFlags` implementation
- ILoader.cs / Loader_1.cs - added GetDebuggerInfoBits and
SetDebuggerInfoBits contract methods with direct target memory
read/write
- IXCLRData.cs - added DebuggerAssemblyControlFlags enum mirroring
cordbpriv.h
- Tests - new tests covering get/set/flag-preservation

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: barosiak

docs/design/datacontracts/Loader.md

@@ -15,9 +15,12 @@ readonly struct ModuleHandle
 [Flags]
 enum ModuleFlags
 {
-    Tenured = 0x00000001, // Set once we know for sure the Module will not be freed until the appdomain itself exits
-    EditAndContinue = 0x00000008,   // Edit and Continue is enabled for this module
-    ReflectionEmit = 0x00000040,    // Reflection.Emit was used to create this module
+    Tenured = 0x1,                      // Set once we know for sure the Module will not be freed until the appdomain itself exits
+    JitOptimizationDisabled = 0x2,      // Cached flag: JIT optimizations are disabled
+    EditAndContinue = 0x8,              // Edit and Continue is enabled for this module
+    ReflectionEmit = 0x40,              // Reflection.Emit was used to create this module
+    ProfDisableOptimizations = 0x80,    // Profiler disabled JIT optimizations
+    EncCapable = 0x200,                 // Cached flag: module is Edit and Continue capable
 }
 
 [Flags]
@@ -90,6 +93,31 @@ TargetPointer GetObjectHandle(TargetPointer loaderAllocatorPointer);
 TargetPointer GetILHeader(ModuleHandle handle, uint token);
 TargetPointer GetDynamicIL(ModuleHandle handle, uint token);
 IReadOnlyDictionary<string, TargetPointer> GetLoaderAllocatorHeaps(TargetPointer loaderAllocatorPointer);
+
+DebuggerAssemblyControlFlags GetDebuggerInfoBits(ModuleHandle handle);
+void SetDebuggerInfoBits(ModuleHandle handle, DebuggerAssemblyControlFlags newBits);
+```
+
+The `DebuggerAssemblyControlFlags` enum is defined as:
+```csharp
+[Flags]
+enum DebuggerAssemblyControlFlags : uint
+{
+    DACF_NONE = 0x00,
+    DACF_ALLOW_JIT_OPTS = 0x02,
+    DACF_ENC_ENABLED = 0x08,
+    DACF_CONTROL_FLAGS_MASK = 0x2E,
+}
+```
+
+The `ClrModifiableAssemblies` enum (from `EEConfig::ModifiableAssemblies`) is defined as:
+```csharp
+enum ClrModifiableAssemblies : uint
+{
+    Unset = 0,
+    None = 1,
+    Debug = 2,
+}
 ```
 
 ## Version 1
@@ -173,6 +201,7 @@ IReadOnlyDictionary<string, TargetPointer> GetLoaderAllocatorHeaps(TargetPointer
 | `DynamicILBlobTable` | `EntrySize` | Size of each table entry |
 | `DynamicILBlobTable` | `EntryMethodToken` | Offset of each entry method token from entry address |
 | `DynamicILBlobTable` | `EntryIL` | Offset of each entry IL from entry address |
+| `EEConfig` | `ModifiableAssemblies` | Controls Edit and Continue support (ClrModifiableAssemblies enum) |
 
 
 
@@ -181,6 +210,7 @@ IReadOnlyDictionary<string, TargetPointer> GetLoaderAllocatorHeaps(TargetPointer
 | --- | --- | --- |
 | `AppDomain` | TargetPointer | Pointer to the global AppDomain |
 | `SystemDomain` | TargetPointer | Pointer to the global SystemDomain |
+| `EEConfig` | TargetPointer | Pointer to the global EEConfig (runtime configuration) |
 
 
 ### Contract Constants:
@@ -189,6 +219,9 @@ IReadOnlyDictionary<string, TargetPointer> GetLoaderAllocatorHeaps(TargetPointer
 | `ASSEMBLY_NOTIFYFLAGS_PROFILER_NOTIFIED` | uint | Flag in Assembly NotifyFlags indicating the Assembly will notify profilers. | `0x1` |
 | `DefaultDomainFriendlyName` | string | Friendly name returned when `AppDomain.FriendlyName` is null (matches native `DEFAULT_DOMAIN_FRIENDLY_NAME`) | `"DefaultDomain"` |
 | `MaxWebcilSections` | ushort | Maximum number of COFF sections supported in a Webcil image (must stay in sync with native `WEBCIL_MAX_SECTIONS`) | `16` |
+| `DebuggerInfoMask` | uint | Mask for the debugger info bits within the Module's transient flags | `0x0000FC00` |
+| `DebuggerInfoShift` | int | Bit shift for the debugger info bits within the Module's transient flags | `10` |
+| `DEBUGGER_ALLOW_JIT_OPTS_PRIV` | uint | Debugger allows JIT optimizations (shifted in transient flags) | `0x00000800` |
 
 Contracts used:
 | Contract Name |
@@ -817,6 +850,36 @@ TargetPointer GetDynamicIL(ModuleHandle handle, uint token)
     Data.DynamicILBlobEntry blobEntry = shashContract.LookupSHash(shash, token);
     return /* blob entry IL address */
 }
+
+DebuggerAssemblyControlFlags GetDebuggerInfoBits(ModuleHandle handle)
+{
+    uint flags = // read Module::Flags at handle.Address + Flags offset
+    return (DebuggerAssemblyControlFlags)((flags & DebuggerInfoMask) >> DebuggerInfoShift);
+}
+
+void SetDebuggerInfoBits(ModuleHandle handle, DebuggerAssemblyControlFlags newBits)
+{
+    uint currentFlags = // read Module::Flags at handle.Address + Flags offset
+    uint debuggerInfoBitsMask = DebuggerInfoMask >> DebuggerInfoShift;
+    uint updated = (currentFlags & ~DebuggerInfoMask) | (((uint)newBits & debuggerInfoBitsMask) << DebuggerInfoShift);
+
+    bool jitOptDisabled = (updated & DEBUGGER_ALLOW_JIT_OPTS_PRIV) == 0 || (updated & PROF_DISABLE_OPTIMIZATIONS) != 0;
+    // Set or clear IS_JIT_OPTIMIZATION_DISABLED accordingly.
+
+    if ((updated & IS_ENC_CAPABLE) != 0)
+    {
+        ClrModifiableAssemblies modifiable = // read EEConfig::ModifiableAssemblies from g_pConfig
+        if (modifiable != None)
+        {
+            bool encRequested = (newBits & DACF_ENC_ENABLED) != 0;
+            bool setEnC = encRequested || (modifiable == Debug && jitOptDisabled);
+            if (setEnC)
+                updated |= IS_EDIT_AND_CONTINUE;
+        }
+    }
+
+    // Write updated flags back to handle.Address + Flags offset
+}
 ```
 
 ### DacEnumerableHash (EETypeHashTable and InstMethodHashTable)

src/coreclr/debug/daccess/dacdbiimpl.cpp

@@ -879,8 +879,6 @@ HRESULT STDMETHODCALLTYPE DacDbiInterfaceImpl::SetCompilerFlags(VMPTR_DomainAsse
             hr = CORDBG_S_NOT_ALL_BITS_SET;
         }
     }
-    // Settings from the debugger take precedence over all other settings.
-    dwBits |= DACF_USER_OVERRIDE;
 
     // set flags. This will write back to the target
     pModule->SetDebuggerInfoBits((DebuggerAssemblyControlFlags)dwBits);

src/coreclr/debug/daccess/task.cpp

@@ -2798,9 +2798,6 @@ ClrDataModule::SetJITCompilerFlags(
                 dwBits |= DACF_ALLOW_JIT_OPTS;
             }
 
-            // Settings from the debugger take precedence over all other settings.
-            dwBits |= DACF_USER_OVERRIDE;
-
             // set flags. This will write back to the target
             m_module->SetDebuggerInfoBits((DebuggerAssemblyControlFlags)dwBits);
 

src/coreclr/inc/cordbpriv.h

@@ -36,7 +36,6 @@ enum DebuggerControlFlag
 
     DBCF_USER_MASK                  = 0x00FF,
     DBCF_GENERATE_DEBUG_CODE        = 0x0001,
-    DBCF_ALLOW_JIT_OPT              = 0x0008,
     DBCF_PROFILER_ENABLED           = 0x0020,
 //    DBCF_ACTIVATE_REMOTE_DEBUGGING  = 0x0040,  Deprecated.  DO NOT USE
 
@@ -50,15 +49,15 @@ enum DebuggerControlFlag
 // Flags used to control the debuggable state of modules and
 // assemblies.
 //
+// [cDAC] [Loader]: Contract depends on DACF_NONE, DACF_ALLOW_JIT_OPTS, DACF_ENC_ENABLED, DACF_CONTROL_FLAGS_MASK.
 enum DebuggerAssemblyControlFlags
 {
     DACF_NONE                       = 0x00,
-    DACF_USER_OVERRIDE              = 0x01,
     DACF_ALLOW_JIT_OPTS             = 0x02,
     DACF_OBSOLETE_TRACK_JIT_INFO    = 0x04, // obsolete in V2.0, we're always tracking.
     DACF_ENC_ENABLED                = 0x08,
     DACF_IGNORE_PDBS                = 0x20,
-    DACF_CONTROL_FLAGS_MASK         = 0x2F,
+    DACF_CONTROL_FLAGS_MASK         = 0x2E,
 
     DACF_PDBS_COPIED                = 0x10,
     DACF_MISC_FLAGS_MASK            = 0x10,

src/coreclr/vm/ceeload.cpp

@@ -516,6 +516,8 @@ void Module::Initialize(AllocMemTracker *pamTracker, LPCWSTR szName)
         m_dwTransientFlags = m_dwTransientFlags | PROF_DISABLE_OPTIMIZATIONS;
     }
 
+    UpdateJitOptimizationDisabledState();
+
     m_pJitInlinerTrackingMap = NULL;
     if (ReJitManager::IsReJITInlineTrackingEnabled())
     {
@@ -538,6 +540,7 @@ void Module::SetDebuggerInfoBits(DebuggerAssemblyControlFlags newBits)
               ~DEBUGGER_INFO_MASK_PRIV) == 0);
 
     SetTransientFlagInterlockedWithMask(newBits << DEBUGGER_INFO_SHIFT_PRIV, DEBUGGER_INFO_MASK_PRIV);
+    UpdateJitOptimizationDisabledState();
 
 #ifdef DEBUGGING_SUPPORTED
     if (IsEditAndContinueCapable())
@@ -611,6 +614,7 @@ Module *Module::Create(Assembly *pAssembly, PEAssembly *pPEAssembly, AllocMemTra
 
         void* pMemory = pamTracker->Track(pAssembly->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(EditAndContinueModule))));
         pModule = new (pMemory) EditAndContinueModule(pAssembly, pPEAssembly);
+        pModule->SetTransientFlagInterlocked(IS_ENC_CAPABLE);
     }
     else
 #endif // FEATURE_METADATA_UPDATER

src/coreclr/vm/ceeload.h

@@ -618,10 +618,10 @@ class Module : public ModuleBase
 
     enum {
         // These are the values set in m_dwTransientFlags.
-        // [cDAC] [Loader]: Contract depends on the values of MODULE_IS_TENURED, IS_EDIT_AND_CONTINUE, and IS_REFLECTION_EMIT.
+        // [cDAC] [Loader]: Contract depends on the values of MODULE_IS_TENURED, IS_EDIT_AND_CONTINUE, IS_REFLECTION_EMIT, IS_JIT_OPTIMIZATION_DISABLED, IS_ENC_CAPABLE, PROF_DISABLE_OPTIMIZATIONS, DEBUGGER_INFO_MASK_PRIV, DEBUGGER_INFO_SHIFT_PRIV.
 
         MODULE_IS_TENURED           = 0x00000001,   // Set once we know for sure the Module will not be freed until the appdomain itself exits
-        // unused                   = 0x00000002,
+        IS_JIT_OPTIMIZATION_DISABLED= 0x00000002,   // Cached result: JIT optimizations are disabled for this module (by debugger or profiler)
         CLASSES_FREED               = 0x00000004,
         IS_EDIT_AND_CONTINUE        = 0x00000008,   // is EnC Enabled for this module
 
@@ -632,12 +632,14 @@ class Module : public ModuleBase
         PROF_DISABLE_OPTIMIZATIONS  = 0x00000080,   // indicates if Profiler disabled JIT optimization event mask was set when loaded
         PROF_DISABLE_INLINING       = 0x00000100,   // indicates if Profiler disabled JIT Inlining event mask was set when loaded
 
+        IS_ENC_CAPABLE              = 0x00000200,   // Cached result of IsEditAndContinueCapable() at Module creation
+
         //
         // Note: The values below must match the ones defined in
         // cordbpriv.h for DebuggerAssemblyControlFlags when shifted
         // right DEBUGGER_INFO_SHIFT bits.
         //
-        DEBUGGER_USER_OVERRIDE_PRIV = 0x00000400,
+        // DEBUGGER_USER_OVERRIDE_PRIV was 0x00000400.  Deprecated.
         DEBUGGER_ALLOW_JIT_OPTS_PRIV= 0x00000800,
         DEBUGGER_TRACK_JIT_INFO_PRIV= 0x00001000,
         DEBUGGER_ENC_ENABLED_PRIV   = 0x00002000,   // this is what was attempted to be set.  IS_EDIT_AND_CONTINUE is actual result.
@@ -651,7 +653,6 @@ class Module : public ModuleBase
         IS_BEING_UNLOADED           = 0x00100000,
     };
 
-    static_assert(DEBUGGER_USER_OVERRIDE_PRIV >> DEBUGGER_INFO_SHIFT_PRIV == DebuggerAssemblyControlFlags::DACF_USER_OVERRIDE);
     static_assert(DEBUGGER_ALLOW_JIT_OPTS_PRIV >> DEBUGGER_INFO_SHIFT_PRIV == DebuggerAssemblyControlFlags::DACF_ALLOW_JIT_OPTS);
     static_assert(DEBUGGER_TRACK_JIT_INFO_PRIV >> DEBUGGER_INFO_SHIFT_PRIV == DebuggerAssemblyControlFlags::DACF_OBSOLETE_TRACK_JIT_INFO);
     static_assert(DEBUGGER_ENC_ENABLED_PRIV >> DEBUGGER_INFO_SHIFT_PRIV == DebuggerAssemblyControlFlags::DACF_ENC_ENABLED);
@@ -938,27 +939,10 @@ class Module : public ModuleBase
 
     BOOL AreJITOptimizationsDisabled() const
     {
-        WRAPPER_NO_CONTRACT;
+        LIMITED_METHOD_CONTRACT;
         SUPPORTS_DAC;
 
-#ifdef DEBUGGING_SUPPORTED
-        // check if debugger has disallowed JIT optimizations
-        auto dwDebuggerBits = GetDebuggerInfoBits();
-        if (!CORDebuggerAllowJITOpts(dwDebuggerBits))
-        {
-            return TRUE;
-        }
-#endif // DEBUGGING_SUPPORTED
-
-#if defined(PROFILING_SUPPORTED) || defined(PROFILING_SUPPORTED_DATA)
-        // check if profiler had disabled JIT optimizations when module was loaded
-        if (m_dwTransientFlags & PROF_DISABLE_OPTIMIZATIONS)
-        {
-            return TRUE;
-        }
-#endif // defined(PROFILING_SUPPORTED) || defined(PROFILING_SUPPORTED_DATA)
-
-        return FALSE;
+        return (m_dwTransientFlags & IS_JIT_OPTIMIZATION_DISABLED) != 0;
     }
 
 #ifdef FEATURE_METADATA_UPDATER
@@ -976,6 +960,17 @@ class Module : public ModuleBase
         SetTransientFlagInterlocked(IS_EDIT_AND_CONTINUE);
     }
 
+    // Recompute and cache the IS_JIT_OPTIMIZATION_DISABLED bit from the debugger and profiler source bits.
+    // Must be called after any change to DEBUGGER_ALLOW_JIT_OPTS_PRIV or PROF_DISABLE_OPTIMIZATIONS.
+    void UpdateJitOptimizationDisabledState()
+    {
+        LIMITED_METHOD_CONTRACT;
+
+        DWORD flags = m_dwTransientFlags;
+        bool disabled = !(flags & DEBUGGER_ALLOW_JIT_OPTS_PRIV) || (flags & PROF_DISABLE_OPTIMIZATIONS);
+        SetTransientFlagInterlockedWithMask(disabled ? IS_JIT_OPTIMIZATION_DISABLED : 0, IS_JIT_OPTIMIZATION_DISABLED);
+    }
+
 public:
     BOOL IsTenured()
     {

src/coreclr/vm/datadescriptor/datadescriptor.inc

@@ -1278,6 +1278,11 @@ CDAC_TYPE_FIELD(AuxiliarySymbolInfo, T_POINTER, Address, offsetof(VMAUXILIARYSYM
 CDAC_TYPE_FIELD(AuxiliarySymbolInfo, T_POINTER, Name, offsetof(VMAUXILIARYSYMBOLDEF, name))
 CDAC_TYPE_END(AuxiliarySymbolInfo)
 
+CDAC_TYPE_BEGIN(EEConfig)
+CDAC_TYPE_INDETERMINATE(EEConfig)
+CDAC_TYPE_FIELD(EEConfig, T_UINT32, ModifiableAssemblies, cdac_data<EEConfig>::ModifiableAssemblies)
+CDAC_TYPE_END(EEConfig)
+
 CDAC_TYPES_END()
 
 CDAC_GLOBALS_BEGIN()
@@ -1328,6 +1333,7 @@ CDAC_GLOBAL_POINTER(SystemDomain, cdac_data<SystemDomain>::SystemDomainPtr)
 CDAC_GLOBAL_POINTER(ThreadStore, &ThreadStore::s_pThreadStore)
 CDAC_GLOBAL_POINTER(FinalizerThread, &::g_pFinalizerThread)
 CDAC_GLOBAL_POINTER(GCThread, &::g_pSuspensionThread)
+CDAC_GLOBAL_POINTER(EEConfig, &::g_pConfig)
 #if defined(DEBUGGING_SUPPORTED) && !defined(TARGET_WASM)
 CDAC_GLOBAL_POINTER(Debugger, &::g_pDebugger)
 CDAC_GLOBAL_POINTER(CLRJitAttachState, &::CLRJitAttachState)

src/coreclr/vm/eeconfig.h

@@ -43,6 +43,7 @@ enum { OPT_BLENDED,
     OPT_RANDOM,
     OPT_DEFAULT = OPT_BLENDED };
 
+// [cDAC] [Loader]: Contract depends on these values.
 enum ClrModifiableAssemblies {
     /* modifiable assemblies are implicitly disabled */
     MODIFIABLE_ASSM_UNSET = 0,
@@ -59,6 +60,7 @@ enum ParseCtl {
 
 class EEConfig
 {
+    friend struct ::cdac_data<EEConfig>;
 public:
     static HRESULT Setup();
 
@@ -715,6 +717,11 @@ class EEConfig
     { return dwSleepOnExit; }
 };
 
+template<>
+struct cdac_data<EEConfig>
+{
+    static constexpr size_t ModifiableAssemblies = offsetof(EEConfig, modifiableAssemblies);
+};
 
 
 #ifdef _DEBUG_IMPL

src/coreclr/vm/vars.hpp

@@ -546,14 +546,8 @@ inline bool CORDebuggerAttached()
     return (g_CORDebuggerControlFlags & DBCF_ATTACHED) && !IsAtProcessExit();
 }
 
-// This only check debugger bits. However JIT optimizations can be disabled by other ways on a module
-// In most cases Module::AreJITOptimizationsDisabled() should be the prefered for checking if JIT optimizations
-// are disabled for a module (it does check both debugger bits and profiler jit deoptimization flag)
 #define CORDebuggerAllowJITOpts(dwDebuggerBits)           \
-    (((dwDebuggerBits) & DACF_ALLOW_JIT_OPTS)             \
-     ||                                                   \
-     ((g_CORDebuggerControlFlags & DBCF_ALLOW_JIT_OPT) && \
-      !((dwDebuggerBits) & DACF_USER_OVERRIDE)))
+    (((dwDebuggerBits) & DACF_ALLOW_JIT_OPTS) != 0)
 
 #define CORDebuggerEnCMode(dwDebuggerBits)                         \
     ((dwDebuggerBits) & DACF_ENC_ENABLED)

src/native/managed/cdac/Microsoft.Diagnostics.DataContractReader.Abstractions/Contracts/ILoader.cs

@@ -19,9 +19,28 @@ public ModuleHandle(TargetPointer address)
 [Flags]
 public enum ModuleFlags
 {
-    Tenured = 0x1,                  // Set once we know for sure the Module will not be freed until the appdomain itself exits
-    EditAndContinue = 0x8,          // Edit and Continue is enabled for this module
-    ReflectionEmit = 0x40,          // Reflection.Emit was used to create this module
+    Tenured = 0x1,                      // Set once we know for sure the Module will not be freed until the appdomain itself exits
+    JitOptimizationDisabled = 0x2,      // Cached flag: JIT optimizations are disabled
+    EditAndContinue = 0x8,              // Edit and Continue is enabled for this module
+    ReflectionEmit = 0x40,              // Reflection.Emit was used to create this module
+    ProfDisableOptimizations = 0x80,    // Profiler disabled JIT optimizations
+    EncCapable = 0x200,                 // Cached flag: module is Edit and Continue capable
+}
+
+[Flags]
+public enum DebuggerAssemblyControlFlags : uint
+{
+    DACF_NONE = 0x00,
+    DACF_ALLOW_JIT_OPTS = 0x02,
+    DACF_ENC_ENABLED = 0x08,
+    DACF_CONTROL_FLAGS_MASK = 0x2E,
+}
+
+public enum ClrModifiableAssemblies : uint
+{
+    Unset = 0,
+    None = 1,
+    Debug = 2,
 }
 
 [Flags]
@@ -98,6 +117,9 @@ public interface ILoader : IContract
     TargetPointer GetObjectHandle(TargetPointer loaderAllocatorPointer) => throw new NotImplementedException();
     TargetPointer GetDynamicIL(ModuleHandle handle, uint token) => throw new NotImplementedException();
     IReadOnlyDictionary<string, TargetPointer> GetLoaderAllocatorHeaps(TargetPointer loaderAllocatorPointer) => throw new NotImplementedException();
+
+    DebuggerAssemblyControlFlags GetDebuggerInfoBits(ModuleHandle handle) => throw new NotImplementedException();
+    void SetDebuggerInfoBits(ModuleHandle handle, DebuggerAssemblyControlFlags newBits) => throw new NotImplementedException();
 }
 
 public readonly struct Loader : ILoader