JMR-dev/ReLibreHardwareMonitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Handle driver restart for NVIDIA GPUs (#2255)

Browse Source 
* Handle driver restart for NVIDIA GPUs.

* PR review

* Update NvidiaGroup.cs

---------

Co-authored-by: PhyxionNL <7643972+PhyxionNL@users.noreply.github.com>
This commit is contained in:
Blacktempel
2026-03-15 11:03:28 +01:00
committed by GitHub
co-authored by PhyxionNL
parent 44d3263b27
commit 1a5a11b12e
2 changed files with 465 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files
LibreHardwareMonitorLib/Hardware/Gpu/NvidiaGpu.cs
+227 -219
View File
@@ -6,6 +6,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Text;
@@ -520,271 +521,278 @@ internal sealed class NvidiaGpu : GenericGpu
public override void Update()
{
if (_d3dDeviceId != null && D3DDisplayDevice.GetDeviceInfoByIdentifier(_d3dDeviceId, out D3DDisplayDevice.D3DDeviceInfo deviceInfo))
try
{
_gpuDedicatedMemoryUsage.Value = 1f * deviceInfo.GpuDedicatedUsed / 1024 / 1024;
_gpuSharedMemoryUsage.Value = 1f * deviceInfo.GpuSharedUsed / 1024 / 1024;
ActivateSensor(_gpuDedicatedMemoryUsage);
ActivateSensor(_gpuSharedMemoryUsage);
foreach (D3DDisplayDevice.D3DDeviceNodeInfo node in deviceInfo.Nodes)
if (_d3dDeviceId != null && D3DDisplayDevice.GetDeviceInfoByIdentifier(_d3dDeviceId, out D3DDisplayDevice.D3DDeviceInfo deviceInfo))
{
long runningTimeDiff = node.RunningTime - _gpuNodeUsagePrevValue[node.Id];
long timeDiff = node.QueryTime.Ticks - _gpuNodeUsagePrevTick[node.Id].Ticks;
_gpuDedicatedMemoryUsage.Value = 1f * deviceInfo.GpuDedicatedUsed / 1024 / 1024;
_gpuSharedMemoryUsage.Value = 1f * deviceInfo.GpuSharedUsed / 1024 / 1024;
ActivateSensor(_gpuDedicatedMemoryUsage);
ActivateSensor(_gpuSharedMemoryUsage);
_gpuNodeUsage[node.Id].Value = 100f * runningTimeDiff / timeDiff;
_gpuNodeUsagePrevValue[node.Id] = node.RunningTime;
_gpuNodeUsagePrevTick[node.Id] = node.QueryTime;
ActivateSensor(_gpuNodeUsage[node.Id]);
}
}
NvApi.NvStatus status;
if (_temperatures is { Length: > 0 })
{
NvApi.NvThermalSettings settings = GetThermalSettings(out status);
// settings.Count is 0 when no valid data available, this happens when you try to read out this value with a high polling interval.
if (status == NvApi.NvStatus.OK && settings.Count > 0)
{
foreach (Sensor sensor in _temperatures)
sensor.Value = settings.Sensor[sensor.Index].CurrentTemp;
}
}
if (_thermalSensorsMask > 0)
{
NvApi.NvThermalSensors thermalSensors = GetThermalSensors(_thermalSensorsMask, out status);
if (status == NvApi.NvStatus.OK)
{
// RTX 50xx series
if (Name.StartsWith("NVIDIA GeForce RTX 50", StringComparison.OrdinalIgnoreCase))
foreach (D3DDisplayDevice.D3DDeviceNodeInfo node in deviceInfo.Nodes)
{
_hotSpotTemperature.Value = 0;
_temperatures[0].Value = thermalSensors.Temperatures[1] / 256.0f;
_memoryJunctionTemperature.Value = thermalSensors.Temperatures[2] / 256.0f;
long runningTimeDiff = node.RunningTime - _gpuNodeUsagePrevValue[node.Id];
long timeDiff = node.QueryTime.Ticks - _gpuNodeUsagePrevTick[node.Id].Ticks;
_gpuNodeUsage[node.Id].Value = 100f * runningTimeDiff / timeDiff;
_gpuNodeUsagePrevValue[node.Id] = node.RunningTime;
_gpuNodeUsagePrevTick[node.Id] = node.QueryTime;
ActivateSensor(_gpuNodeUsage[node.Id]);
}
// RTX 40xx series
else if (Name.StartsWith("NVIDIA GeForce RTX 40", StringComparison.OrdinalIgnoreCase))
}
NvApi.NvStatus status;
if (_temperatures is { Length: > 0 })
{
NvApi.NvThermalSettings settings = GetThermalSettings(out status);
// settings.Count is 0 when no valid data available, this happens when you try to read out this value with a high polling interval.
if (status == NvApi.NvStatus.OK && settings.Count > 0)
{
_hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
_memoryJunctionTemperature.Value = thermalSensors.Temperatures[7] / 256.0f;
foreach (Sensor sensor in _temperatures)
sensor.Value = settings.Sensor[sensor.Index].CurrentTemp;
}
}
if (_thermalSensorsMask > 0)
{
NvApi.NvThermalSensors thermalSensors = GetThermalSensors(_thermalSensorsMask, out status);
if (status == NvApi.NvStatus.OK)
{
// RTX 50xx series
if (Name.StartsWith("NVIDIA GeForce RTX 50", StringComparison.OrdinalIgnoreCase))
{
_hotSpotTemperature.Value = 0;
_temperatures[0].Value = thermalSensors.Temperatures[1] / 256.0f;
_memoryJunctionTemperature.Value = thermalSensors.Temperatures[2] / 256.0f;
}
// RTX 40xx series
else if (Name.StartsWith("NVIDIA GeForce RTX 40", StringComparison.OrdinalIgnoreCase))
{
_hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
_memoryJunctionTemperature.Value = thermalSensors.Temperatures[7] / 256.0f;
}
else
{
_hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
_memoryJunctionTemperature.Value = thermalSensors.Temperatures[9] / 256.0f;
}
}
if (_hotSpotTemperature.Value != 0)
ActivateSensor(_hotSpotTemperature);
if (_memoryJunctionTemperature.Value != 0)
ActivateSensor(_memoryJunctionTemperature);
}
else
{
_hotSpotTemperature.Value = null;
_memoryJunctionTemperature.Value = null;
}
if (_clocks is { Length: > 0 })
{
NvApi.NvGpuClockFrequencies clockFrequencies = GetClockFrequencies(out status);
if (status == NvApi.NvStatus.OK)
{
int current = 0;
for (int i = 0; i < clockFrequencies.Clocks.Length; i++)
{
NvApi.NvGpuClockFrequenciesDomain clock = clockFrequencies.Clocks[i];
if (clock.IsPresent && Enum.IsDefined(typeof(NvApi.NvGpuPublicClockId), i))
_clocks[current++].Value = clock.Frequency / 1000f;
}
}
}
if (_fans is { Length: > 0 })
{
NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0)
{
for (int i = 0; i < fanCoolers.Count; i++)
{
NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
_fans[i].Value = item.CurrentRpm;
}
}
else
{
_hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
_memoryJunctionTemperature.Value = thermalSensors.Temperatures[9] / 256.0f;
int tachReading = GetTachReading(out status);
if (status == NvApi.NvStatus.OK)
_fans[0].Value = tachReading;
}
}
if (_hotSpotTemperature.Value != 0)
ActivateSensor(_hotSpotTemperature);
if (_memoryJunctionTemperature.Value != 0)
ActivateSensor(_memoryJunctionTemperature);
}
else
{
_hotSpotTemperature.Value = null;
_memoryJunctionTemperature.Value = null;
}
if (_clocks is { Length: > 0 })
{
NvApi.NvGpuClockFrequencies clockFrequencies = GetClockFrequencies(out status);
if (status == NvApi.NvStatus.OK)
if (_controls is { Length: > 0 })
{
int current = 0;
for (int i = 0; i < clockFrequencies.Clocks.Length; i++)
NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0 && fanCoolers.Count == _controls.Length)
{
NvApi.NvGpuClockFrequenciesDomain clock = clockFrequencies.Clocks[i];
if (clock.IsPresent && Enum.IsDefined(typeof(NvApi.NvGpuPublicClockId), i))
_clocks[current++].Value = clock.Frequency / 1000f;
}
}
}
if (_fans is { Length: > 0 })
{
NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0)
{
for (int i = 0; i < fanCoolers.Count; i++)
{
NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
_fans[i].Value = item.CurrentRpm;
}
}
else
{
int tachReading = GetTachReading(out status);
if (status == NvApi.NvStatus.OK)
_fans[0].Value = tachReading;
}
}
if (_controls is { Length: > 0 })
{
NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0 && fanCoolers.Count == _controls.Length)
{
for (int i = 0; i < fanCoolers.Count; i++)
{
NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
if (Array.Find(_controls, c => c.Index == item.CoolerId) is { } control)
control.Value = item.CurrentLevel;
}
}
else
{
NvApi.NvCoolerSettings coolerSettings = GetCoolerSettings(out status);
if (status == NvApi.NvStatus.OK && coolerSettings.Count > 0)
{
for (int i = 0; i < coolerSettings.Count; i++)
for (int i = 0; i < fanCoolers.Count; i++)
{
NvApi.NvCooler cooler = coolerSettings.Cooler[i];
_controls[i].Value = cooler.CurrentLevel;
NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
if (Array.Find(_controls, c => c.Index == item.CoolerId) is { } control)
control.Value = item.CurrentLevel;
}
}
else
{
NvApi.NvCoolerSettings coolerSettings = GetCoolerSettings(out status);
if (status == NvApi.NvStatus.OK && coolerSettings.Count > 0)
{
for (int i = 0; i < coolerSettings.Count; i++)
{
NvApi.NvCooler cooler = coolerSettings.Cooler[i];
_controls[i].Value = cooler.CurrentLevel;
}
}
}
}
}
if (_loads is { Length: > 0 })
{
NvApi.NvDynamicPStatesInfo pStatesInfo = GetDynamicPstatesInfoEx(out status);
if (status == NvApi.NvStatus.OK)
if (_loads is { Length: > 0 })
{
for (int index = 0; index < pStatesInfo.Utilizations.Length; index++)
{
NvApi.NvDynamicPState load = pStatesInfo.Utilizations[index];
if (load.IsPresent && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
_loads[index].Value = load.Percentage;
}
}
else
{
NvApi.NvUsages usages = GetUsages(out status);
NvApi.NvDynamicPStatesInfo pStatesInfo = GetDynamicPstatesInfoEx(out status);
if (status == NvApi.NvStatus.OK)
{
for (int index = 0; index < usages.Entries.Length; index++)
for (int index = 0; index < pStatesInfo.Utilizations.Length; index++)
{
NvApi.NvUsagesEntry load = usages.Entries[index];
if (load.IsPresent > 0 && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
NvApi.NvDynamicPState load = pStatesInfo.Utilizations[index];
if (load.IsPresent && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
_loads[index].Value = load.Percentage;
}
}
}
}
if (_powers is { Length: > 0 })
{
NvApi.NvPowerTopology powerTopology = GetPowerTopology(out status);
if (status == NvApi.NvStatus.OK && powerTopology.Count > 0)
{
for (int i = 0; i < powerTopology.Count; i++)
else
{
NvApi.NvPowerTopologyEntry entry = powerTopology.Entries[i];
_powers[i].Value = entry.PowerUsage / 1000f;
NvApi.NvUsages usages = GetUsages(out status);
if (status == NvApi.NvStatus.OK)
{
for (int index = 0; index < usages.Entries.Length; index++)
{
NvApi.NvUsagesEntry load = usages.Entries[index];
if (load.IsPresent > 0 && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
_loads[index].Value = load.Percentage;
}
}
}
}
}
if (_coreVoltage != null)
{
var voltageSensor = GetVoltRailsStatus(out status);
if (status == NvApi.NvStatus.OK)
if (_powers is { Length: > 0 })
{
var microvolts = ((ulong)voltageSensor.CoreMicrovoltsHigh << 32) | voltageSensor.CoreMicrovolts;
_coreVoltage.Value = microvolts == 0 ? 0 : microvolts / 1_000_000f;
NvApi.NvPowerTopology powerTopology = GetPowerTopology(out status);
if (status == NvApi.NvStatus.OK && powerTopology.Count > 0)
{
for (int i = 0; i < powerTopology.Count; i++)
{
NvApi.NvPowerTopologyEntry entry = powerTopology.Entries[i];
_powers[i].Value = entry.PowerUsage / 1000f;
}
}
}
}
if (NvApi.NvAPI_GPU_GetMemoryInfoEx != null || _displayHandle != null)
{
uint free = 0;
uint total = 0;
//Size in bytes
NvApi.NvMemoryInfoEx memoryInfoEx = GetMemoryInfoEx(out status);
if (status == NvApi.NvStatus.OK)
if (_coreVoltage != null)
{
free = (uint)(memoryInfoEx.CurrentAvailableDedicatedVideoMemory / 1024);
total = (uint)(memoryInfoEx.DedicatedVideoMemory / 1024);
}
else
{
//Size in kilobytes, fallback for older drivers
NvApi.NvMemoryInfo memoryInfo = GetMemoryInfo(out status);
var voltageSensor = GetVoltRailsStatus(out status);
if (status == NvApi.NvStatus.OK)
{
free = memoryInfo.CurrentAvailableDedicatedVideoMemory;
total = memoryInfo.DedicatedVideoMemory;
var microvolts = ((ulong)voltageSensor.CoreMicrovoltsHigh << 32) | voltageSensor.CoreMicrovolts;
_coreVoltage.Value = microvolts == 0 ? 0 : microvolts / 1_000_000f;
}
}
if (status == NvApi.NvStatus.OK)
if (NvApi.NvAPI_GPU_GetMemoryInfoEx != null || _displayHandle != null)
{
_memoryTotal.Value = total / 1024;
ActivateSensor(_memoryTotal);
uint free = 0;
uint total = 0;
_memoryFree.Value = free / 1024;
ActivateSensor(_memoryFree);
//Size in bytes
NvApi.NvMemoryInfoEx memoryInfoEx = GetMemoryInfoEx(out status);
if (status == NvApi.NvStatus.OK)
{
free = (uint)(memoryInfoEx.CurrentAvailableDedicatedVideoMemory / 1024);
total = (uint)(memoryInfoEx.DedicatedVideoMemory / 1024);
}
else
{
//Size in kilobytes, fallback for older drivers
NvApi.NvMemoryInfo memoryInfo = GetMemoryInfo(out status);
if (status == NvApi.NvStatus.OK)
{
free = memoryInfo.CurrentAvailableDedicatedVideoMemory;
total = memoryInfo.DedicatedVideoMemory;
}
}
_memoryUsed.Value = (total - free) / 1024;
ActivateSensor(_memoryUsed);
if (status == NvApi.NvStatus.OK)
{
_memoryTotal.Value = total / 1024;
ActivateSensor(_memoryTotal);
_memoryLoad.Value = ((float)(total - free) / total) * 100;
ActivateSensor(_memoryLoad);
_memoryFree.Value = free / 1024;
ActivateSensor(_memoryFree);
_memoryUsed.Value = (total - free) / 1024;
ActivateSensor(_memoryUsed);
_memoryLoad.Value = ((float)(total - free) / total) * 100;
ActivateSensor(_memoryLoad);
}
}
if (NvidiaML.IsAvailable && _nvmlDevice.HasValue)
{
int? result = NvidiaML.NvmlDeviceGetPowerUsage(_nvmlDevice.Value);
if (result.HasValue)
{
_powerUsage.Value = result.Value / 1000f;
ActivateSensor(_powerUsage);
}
// In MB/s, throughput sensors are passed as in KB/s.
uint? rx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.RxBytes);
if (rx.HasValue)
{
_pcieThroughputRx.Value = rx * 1024;
ActivateSensor(_pcieThroughputRx);
}
uint? tx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.TxBytes);
if (tx.HasValue)
{
_pcieThroughputTx.Value = tx * 1024;
ActivateSensor(_pcieThroughputTx);
}
}
// Astral specific
if (_12VHPwrPinCurrentSensors is { Length: > 0 } && TryReadAstral12VHPwrPinSensors(out ushort[] pinSensorValues))
{
float connectorCurrent = 0;
float connectorPower = 0;
for (int i = 0; i < 6; i++)
{
float current = pinSensorValues[i * 2] / 1000f;
float voltage = pinSensorValues[i * 2 + 1] / 1000f;
_12VHPwrPinVoltageSensors[i].Value = voltage;
_12VHPwrPinCurrentSensors[i].Value = current;
_12VHPwrPinPowerSensors[i].Value = voltage * current;
connectorCurrent += current;
connectorPower += voltage * current;
}
_12VHPwrConnectorCurrentSensor.Value = connectorCurrent;
_12VHPwrConnectorPowerSensor.Value = connectorPower;
}
}
if (NvidiaML.IsAvailable && _nvmlDevice.HasValue)
catch (Exception e)
{
int? result = NvidiaML.NvmlDeviceGetPowerUsage(_nvmlDevice.Value);
if (result.HasValue)
{
_powerUsage.Value = result.Value / 1000f;
ActivateSensor(_powerUsage);
}
// In MB/s, throughput sensors are passed as in KB/s.
uint? rx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.RxBytes);
if (rx.HasValue)
{
_pcieThroughputRx.Value = rx * 1024;
ActivateSensor(_pcieThroughputRx);
}
uint? tx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.TxBytes);
if (tx.HasValue)
{
_pcieThroughputTx.Value = tx * 1024;
ActivateSensor(_pcieThroughputTx);
}
}
// Astral specific
if (_12VHPwrPinCurrentSensors is { Length: > 0 } && TryReadAstral12VHPwrPinSensors(out ushort[] pinSensorValues))
{
float connectorCurrent = 0;
float connectorPower = 0;
for (int i = 0; i < 6; i++)
{
float current = pinSensorValues[i * 2] / 1000f;
float voltage = pinSensorValues[i * 2 + 1] / 1000f;
_12VHPwrPinVoltageSensors[i].Value = voltage;
_12VHPwrPinCurrentSensors[i].Value = current;
_12VHPwrPinPowerSensors[i].Value = voltage * current;
connectorCurrent += current;
connectorPower += voltage * current;
}
_12VHPwrConnectorCurrentSensor.Value = connectorCurrent;
_12VHPwrConnectorPowerSensor.Value = connectorPower;
Debug.WriteLine($"{nameof(NvidiaGpu)} {nameof(Update)} failed for '{Name}' ({Identifier}): {e}");
}
}
LibreHardwareMonitorLib/Hardware/Gpu/NvidiaGroup.cs
+238 -63
View File
@@ -4,100 +4,275 @@
// Partial Copyright (C) Michael Möller <mmoeller@openhardwaremonitor.org> and Contributors.
// All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using LibreHardwareMonitor.Interop;
namespace LibreHardwareMonitor.Hardware.Gpu;
internal class NvidiaGroup : IGroup
internal class NvidiaGroup : IGroup, IHardwareChanged
{
private readonly List<Hardware> _hardware = new();
private readonly Dictionary<NvApi.NvPhysicalGpuHandle, NvidiaGpu> _hardwareByHandle = new();
private readonly List<Hardware> _hardware = [];
private readonly StringBuilder _report = new();
private readonly ISettings _settings;
private CancellationTokenSource _cancellationTokenSource;
private bool _disposed;
private Task _monitorTask;
private bool _nvidiaWasAvailable;
public NvidiaGroup(ISettings settings)
{
NvApi.Initialize();
if (!NvApi.IsAvailable)
return;
_settings = settings;
_report.AppendLine("NvApi");
_report.AppendLine();
if (NvApi.NvAPI_GetInterfaceVersionString(out string version) == NvApi.NvStatus.OK)
if (Software.OperatingSystem.IsUnix)
{
_report.Append("Version: ");
_report.AppendLine(version);
}
NvApi.NvPhysicalGpuHandle[] handles = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
if (NvApi.NvAPI_EnumPhysicalGPUs == null)
{
_report.AppendLine("Error: NvAPI_EnumPhysicalGPUs not available");
_report.AppendLine("Status: Not supported on Unix for NvApi group");
_report.AppendLine();
return;
}
NvApi.NvStatus status = NvApi.NvAPI_EnumPhysicalGPUs(handles, out int count);
if (status != NvApi.NvStatus.OK)
{
_report.AppendLine("Status: " + status);
_report.AppendLine();
return;
}
IDictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> displayHandles = new Dictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle>();
if (NvApi.NvAPI_EnumNvidiaDisplayHandle != null && NvApi.NvAPI_GetPhysicalGPUsFromDisplay != null)
{
status = NvApi.NvStatus.OK;
int i = 0;
while (status == NvApi.NvStatus.OK)
{
NvApi.NvDisplayHandle displayHandle = new();
status = NvApi.NvAPI_EnumNvidiaDisplayHandle(i, ref displayHandle);
i++;
if (status == NvApi.NvStatus.OK)
{
NvApi.NvPhysicalGpuHandle[] handlesFromDisplay = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
if (NvApi.NvAPI_GetPhysicalGPUsFromDisplay(displayHandle, handlesFromDisplay, out uint countFromDisplay) == NvApi.NvStatus.OK)
{
for (int j = 0; j < countFromDisplay; j++)
{
if (!displayHandles.ContainsKey(handlesFromDisplay[j]))
displayHandles.Add(handlesFromDisplay[j], displayHandle);
}
}
}
}
}
_report.Append("Number of GPUs: ");
_report.AppendLine(count.ToString(CultureInfo.InvariantCulture));
for (int i = 0; i < count; i++)
{
displayHandles.TryGetValue(handles[i], out NvApi.NvDisplayHandle displayHandle);
_hardware.Add(new NvidiaGpu(i, handles[i], displayHandle, settings));
}
_report.AppendLine();
RefreshHardware(raiseEvents: false);
StartMonitorTask();
}
public IReadOnlyList<IHardware> Hardware => _hardware;
public event HardwareEventHandler HardwareAdded;
public string GetReport()
public event HardwareEventHandler HardwareRemoved;
public IReadOnlyList<IHardware> Hardware
{
return _report.ToString();
get { return _hardware; }
}
public string GetReport() => _report.ToString();
public void Close()
{
foreach (Hardware gpu in _hardware)
_disposed = true;
_cancellationTokenSource?.Cancel();
try
{
_monitorTask?.Wait(TimeSpan.FromSeconds(1));
}
catch (AggregateException)
{
// ignored
}
_cancellationTokenSource?.Dispose();
_cancellationTokenSource = null;
_monitorTask = null;
var hardwareToClose = _hardware.ToList();
_hardware.Clear();
_hardwareByHandle.Clear();
foreach (Hardware gpu in hardwareToClose)
gpu.Close();
NvidiaML.Close();
}
private void StartMonitorTask()
{
CancellationTokenSource cts = new();
_cancellationTokenSource = cts;
CancellationToken token = cts.Token;
_monitorTask = Task.Run(async () =>
{
while (!token.IsCancellationRequested)
{
try
{
await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
break;
}
if (_disposed)
{
break;
}
RefreshHardware(raiseEvents: true);
}
},
token);
}
private void RefreshHardware(bool raiseEvents)
{
if (_disposed)
{
return;
}
bool isAvailable = TryEnumerateGpus(out NvApi.NvPhysicalGpuHandle[] handles, out int count);
if (!isAvailable)
{
if (_nvidiaWasAvailable)
{
NvidiaML.Close();
}
_nvidiaWasAvailable = false;
RemoveAllHardware(raiseEvents);
return;
}
// Driver was unavailable and came back: force NVML re-init
if (!_nvidiaWasAvailable)
{
NvidiaML.Close();
if (NvApi.NvAPI_GetInterfaceVersionString(out string version) == NvApi.NvStatus.OK)
{
_report.Append("Version: ");
_report.AppendLine(version);
}
_report.Append("Number of GPUs: ");
_report.AppendLine(count.ToString(CultureInfo.InvariantCulture));
_report.AppendLine();
}
_nvidiaWasAvailable = true;
IDictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> displayHandles = GetDisplayHandles();
HashSet<NvApi.NvPhysicalGpuHandle> currentSet = [.. handles.Take(count)];
List<Hardware> removed = [];
foreach (KeyValuePair<NvApi.NvPhysicalGpuHandle, NvidiaGpu> pair in _hardwareByHandle.ToList())
{
if (!currentSet.Contains(pair.Key))
{
_hardwareByHandle.Remove(pair.Key);
_hardware.Remove(pair.Value);
removed.Add(pair.Value);
}
}
for (int i = 0; i < count; ++i)
{
NvApi.NvPhysicalGpuHandle handle = handles[i];
if (_hardwareByHandle.ContainsKey(handle))
continue;
displayHandles.TryGetValue(handle, out NvApi.NvDisplayHandle displayHandle);
NvidiaGpu gpu = new(i, handle, displayHandle, _settings);
_hardwareByHandle.Add(handle, gpu);
_hardware.Add(gpu);
if (raiseEvents)
HardwareAdded?.Invoke(gpu);
}
foreach (Hardware gpu in removed)
{
if (raiseEvents)
HardwareRemoved?.Invoke(gpu);
gpu.Close();
}
}
private void RemoveAllHardware(bool raiseEvents)
{
if (_hardware.Count == 0)
return;
List<Hardware> removed = _hardware.ToList();
_hardware.Clear();
_hardwareByHandle.Clear();
foreach (Hardware gpu in removed)
{
if (raiseEvents)
{
HardwareRemoved?.Invoke(gpu);
}
gpu.Close();
}
}
private static IDictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> GetDisplayHandles()
{
Dictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> displayHandles = [];
if (NvApi.NvAPI_EnumNvidiaDisplayHandle == null || NvApi.NvAPI_GetPhysicalGPUsFromDisplay == null)
{
return displayHandles;
}
NvApi.NvStatus status = NvApi.NvStatus.OK;
int i = 0;
while (status == NvApi.NvStatus.OK)
{
NvApi.NvDisplayHandle displayHandle = new();
status = NvApi.NvAPI_EnumNvidiaDisplayHandle(i, ref displayHandle);
i++;
if (status != NvApi.NvStatus.OK)
{
continue;
}
NvApi.NvPhysicalGpuHandle[] handlesFromDisplay = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
if (NvApi.NvAPI_GetPhysicalGPUsFromDisplay(displayHandle, handlesFromDisplay, out uint countFromDisplay) != NvApi.NvStatus.OK)
{
continue;
}
for (int j = 0; j < countFromDisplay; j++)
{
if (!displayHandles.ContainsKey(handlesFromDisplay[j]))
{
displayHandles.Add(handlesFromDisplay[j], displayHandle);
}
}
}
return displayHandles;
}
private static bool TryEnumerateGpus(out NvApi.NvPhysicalGpuHandle[] handles, out int count)
{
handles = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
count = 0;
NvApi.Initialize();
if (!NvApi.IsAvailable || NvApi.NvAPI_EnumPhysicalGPUs == null)
{
return false;
}
NvApi.NvStatus status = NvApi.NvAPI_EnumPhysicalGPUs(handles, out count);
return status == NvApi.NvStatus.OK && count > 0;
}
}