Handle driver restart for NVIDIA GPUs (#2255)

* Handle driver restart for NVIDIA GPUs.

* PR review

* Update NvidiaGroup.cs

---------

Co-authored-by: PhyxionNL <7643972+PhyxionNL@users.noreply.github.com>

LibreHardwareMonitorLib/Hardware/Gpu/NvidiaGpu.cs

@@ -6,6 +6,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Text;
@@ -520,271 +521,278 @@ internal sealed class NvidiaGpu : GenericGpu
 
     public override void Update()
     {
-        if (_d3dDeviceId != null && D3DDisplayDevice.GetDeviceInfoByIdentifier(_d3dDeviceId, out D3DDisplayDevice.D3DDeviceInfo deviceInfo))
+        try
         {
-            _gpuDedicatedMemoryUsage.Value = 1f * deviceInfo.GpuDedicatedUsed / 1024 / 1024;
+            if (_d3dDeviceId != null && D3DDisplayDevice.GetDeviceInfoByIdentifier(_d3dDeviceId, out D3DDisplayDevice.D3DDeviceInfo deviceInfo))
-            _gpuSharedMemoryUsage.Value = 1f * deviceInfo.GpuSharedUsed / 1024 / 1024;
-            ActivateSensor(_gpuDedicatedMemoryUsage);
-            ActivateSensor(_gpuSharedMemoryUsage);
-
-            foreach (D3DDisplayDevice.D3DDeviceNodeInfo node in deviceInfo.Nodes)
             {
-                long runningTimeDiff = node.RunningTime - _gpuNodeUsagePrevValue[node.Id];
+                _gpuDedicatedMemoryUsage.Value = 1f * deviceInfo.GpuDedicatedUsed / 1024 / 1024;
-                long timeDiff = node.QueryTime.Ticks - _gpuNodeUsagePrevTick[node.Id].Ticks;
+                _gpuSharedMemoryUsage.Value = 1f * deviceInfo.GpuSharedUsed / 1024 / 1024;
+                ActivateSensor(_gpuDedicatedMemoryUsage);
+                ActivateSensor(_gpuSharedMemoryUsage);
 
-                _gpuNodeUsage[node.Id].Value = 100f * runningTimeDiff / timeDiff;
+                foreach (D3DDisplayDevice.D3DDeviceNodeInfo node in deviceInfo.Nodes)
-                _gpuNodeUsagePrevValue[node.Id] = node.RunningTime;
-                _gpuNodeUsagePrevTick[node.Id] = node.QueryTime;
-                ActivateSensor(_gpuNodeUsage[node.Id]);
-            }
-        }
-
-        NvApi.NvStatus status;
-
-        if (_temperatures is { Length: > 0 })
-        {
-            NvApi.NvThermalSettings settings = GetThermalSettings(out status);
-            // settings.Count is 0 when no valid data available, this happens when you try to read out this value with a high polling interval.
-            if (status == NvApi.NvStatus.OK && settings.Count > 0)
-            {
-                foreach (Sensor sensor in _temperatures)
-                    sensor.Value = settings.Sensor[sensor.Index].CurrentTemp;
-            }
-        }
-
-        if (_thermalSensorsMask > 0)
-        {
-            NvApi.NvThermalSensors thermalSensors = GetThermalSensors(_thermalSensorsMask, out status);
-
-            if (status == NvApi.NvStatus.OK)
-            {
-                // RTX 50xx series
-                if (Name.StartsWith("NVIDIA GeForce RTX 50", StringComparison.OrdinalIgnoreCase))
                 {
-                    _hotSpotTemperature.Value = 0;
+                    long runningTimeDiff = node.RunningTime - _gpuNodeUsagePrevValue[node.Id];
-                    _temperatures[0].Value = thermalSensors.Temperatures[1] / 256.0f;
+                    long timeDiff = node.QueryTime.Ticks - _gpuNodeUsagePrevTick[node.Id].Ticks;
-                    _memoryJunctionTemperature.Value = thermalSensors.Temperatures[2] / 256.0f;
+
+                    _gpuNodeUsage[node.Id].Value = 100f * runningTimeDiff / timeDiff;
+                    _gpuNodeUsagePrevValue[node.Id] = node.RunningTime;
+                    _gpuNodeUsagePrevTick[node.Id] = node.QueryTime;
+                    ActivateSensor(_gpuNodeUsage[node.Id]);
                 }
-                // RTX 40xx series
+            }
-                else if (Name.StartsWith("NVIDIA GeForce RTX 40", StringComparison.OrdinalIgnoreCase))
+
+            NvApi.NvStatus status;
+
+            if (_temperatures is { Length: > 0 })
+            {
+                NvApi.NvThermalSettings settings = GetThermalSettings(out status);
+                // settings.Count is 0 when no valid data available, this happens when you try to read out this value with a high polling interval.
+                if (status == NvApi.NvStatus.OK && settings.Count > 0)
                 {
-                    _hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
+                    foreach (Sensor sensor in _temperatures)
-                    _memoryJunctionTemperature.Value = thermalSensors.Temperatures[7] / 256.0f;
+                        sensor.Value = settings.Sensor[sensor.Index].CurrentTemp;
+                }
+            }
+
+            if (_thermalSensorsMask > 0)
+            {
+                NvApi.NvThermalSensors thermalSensors = GetThermalSensors(_thermalSensorsMask, out status);
+
+                if (status == NvApi.NvStatus.OK)
+                {
+                    // RTX 50xx series
+                    if (Name.StartsWith("NVIDIA GeForce RTX 50", StringComparison.OrdinalIgnoreCase))
+                    {
+                        _hotSpotTemperature.Value = 0;
+                        _temperatures[0].Value = thermalSensors.Temperatures[1] / 256.0f;
+                        _memoryJunctionTemperature.Value = thermalSensors.Temperatures[2] / 256.0f;
+                    }
+                    // RTX 40xx series
+                    else if (Name.StartsWith("NVIDIA GeForce RTX 40", StringComparison.OrdinalIgnoreCase))
+                    {
+                        _hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
+                        _memoryJunctionTemperature.Value = thermalSensors.Temperatures[7] / 256.0f;
+                    }
+                    else
+                    {
+                        _hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
+                        _memoryJunctionTemperature.Value = thermalSensors.Temperatures[9] / 256.0f;
+                    }
+                }
+
+                if (_hotSpotTemperature.Value != 0)
+                    ActivateSensor(_hotSpotTemperature);
+
+                if (_memoryJunctionTemperature.Value != 0)
+                    ActivateSensor(_memoryJunctionTemperature);
+            }
+            else
+            {
+                _hotSpotTemperature.Value = null;
+                _memoryJunctionTemperature.Value = null;
+            }
+
+            if (_clocks is { Length: > 0 })
+            {
+                NvApi.NvGpuClockFrequencies clockFrequencies = GetClockFrequencies(out status);
+                if (status == NvApi.NvStatus.OK)
+                {
+                    int current = 0;
+                    for (int i = 0; i < clockFrequencies.Clocks.Length; i++)
+                    {
+                        NvApi.NvGpuClockFrequenciesDomain clock = clockFrequencies.Clocks[i];
+                        if (clock.IsPresent && Enum.IsDefined(typeof(NvApi.NvGpuPublicClockId), i))
+                            _clocks[current++].Value = clock.Frequency / 1000f;
+                    }
+                }
+            }
+
+            if (_fans is { Length: > 0 })
+            {
+                NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
+                if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0)
+                {
+                    for (int i = 0; i < fanCoolers.Count; i++)
+                    {
+                        NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
+                        _fans[i].Value = item.CurrentRpm;
+                    }
                 }
                 else
                 {
-                    _hotSpotTemperature.Value = thermalSensors.Temperatures[1] / 256.0f;
+                    int tachReading = GetTachReading(out status);
-                    _memoryJunctionTemperature.Value = thermalSensors.Temperatures[9] / 256.0f;
+                    if (status == NvApi.NvStatus.OK)
+                        _fans[0].Value = tachReading;
                 }
             }
 
-            if (_hotSpotTemperature.Value != 0)
+            if (_controls is { Length: > 0 })
-                ActivateSensor(_hotSpotTemperature);
-
-            if (_memoryJunctionTemperature.Value != 0)
-                ActivateSensor(_memoryJunctionTemperature);
-        }
-        else
-        {
-            _hotSpotTemperature.Value = null;
-            _memoryJunctionTemperature.Value = null;
-        }
-
-        if (_clocks is { Length: > 0 })
-        {
-            NvApi.NvGpuClockFrequencies clockFrequencies = GetClockFrequencies(out status);
-            if (status == NvApi.NvStatus.OK)
             {
-                int current = 0;
+                NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
-                for (int i = 0; i < clockFrequencies.Clocks.Length; i++)
+                if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0 && fanCoolers.Count == _controls.Length)
                 {
-                    NvApi.NvGpuClockFrequenciesDomain clock = clockFrequencies.Clocks[i];
+                    for (int i = 0; i < fanCoolers.Count; i++)
-                    if (clock.IsPresent && Enum.IsDefined(typeof(NvApi.NvGpuPublicClockId), i))
-                        _clocks[current++].Value = clock.Frequency / 1000f;
-                }
-            }
-        }
-
-        if (_fans is { Length: > 0 })
-        {
-            NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
-            if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0)
-            {
-                for (int i = 0; i < fanCoolers.Count; i++)
-                {
-                    NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
-                    _fans[i].Value = item.CurrentRpm;
-                }
-            }
-            else
-            {
-                int tachReading = GetTachReading(out status);
-                if (status == NvApi.NvStatus.OK)
-                    _fans[0].Value = tachReading;
-            }
-        }
-
-        if (_controls is { Length: > 0 })
-        {
-            NvApi.NvFanCoolersStatus fanCoolers = GetFanCoolersStatus(out status);
-            if (status == NvApi.NvStatus.OK && fanCoolers.Count > 0 && fanCoolers.Count == _controls.Length)
-            {
-                for (int i = 0; i < fanCoolers.Count; i++)
-                {
-                    NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
-
-                    if (Array.Find(_controls, c => c.Index == item.CoolerId) is { } control)
-                        control.Value = item.CurrentLevel;
-                }
-            }
-            else
-            {
-                NvApi.NvCoolerSettings coolerSettings = GetCoolerSettings(out status);
-                if (status == NvApi.NvStatus.OK && coolerSettings.Count > 0)
-                {
-                    for (int i = 0; i < coolerSettings.Count; i++)
                     {
-                        NvApi.NvCooler cooler = coolerSettings.Cooler[i];
+                        NvApi.NvFanCoolersStatusItem item = fanCoolers.Items[i];
-                        _controls[i].Value = cooler.CurrentLevel;
+
+                        if (Array.Find(_controls, c => c.Index == item.CoolerId) is { } control)
+                            control.Value = item.CurrentLevel;
+                    }
+                }
+                else
+                {
+                    NvApi.NvCoolerSettings coolerSettings = GetCoolerSettings(out status);
+                    if (status == NvApi.NvStatus.OK && coolerSettings.Count > 0)
+                    {
+                        for (int i = 0; i < coolerSettings.Count; i++)
+                        {
+                            NvApi.NvCooler cooler = coolerSettings.Cooler[i];
+                            _controls[i].Value = cooler.CurrentLevel;
+                        }
                     }
                 }
             }
-        }
 
-        if (_loads is { Length: > 0 })
+            if (_loads is { Length: > 0 })
-        {
-            NvApi.NvDynamicPStatesInfo pStatesInfo = GetDynamicPstatesInfoEx(out status);
-            if (status == NvApi.NvStatus.OK)
             {
-                for (int index = 0; index < pStatesInfo.Utilizations.Length; index++)
+                NvApi.NvDynamicPStatesInfo pStatesInfo = GetDynamicPstatesInfoEx(out status);
-                {
-                    NvApi.NvDynamicPState load = pStatesInfo.Utilizations[index];
-                    if (load.IsPresent && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
-                        _loads[index].Value = load.Percentage;
-                }
-            }
-            else
-            {
-                NvApi.NvUsages usages = GetUsages(out status);
                 if (status == NvApi.NvStatus.OK)
                 {
-                    for (int index = 0; index < usages.Entries.Length; index++)
+                    for (int index = 0; index < pStatesInfo.Utilizations.Length; index++)
                     {
-                        NvApi.NvUsagesEntry load = usages.Entries[index];
+                        NvApi.NvDynamicPState load = pStatesInfo.Utilizations[index];
-                        if (load.IsPresent > 0 && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
+                        if (load.IsPresent && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
                             _loads[index].Value = load.Percentage;
                     }
                 }
-            }
+                else
-        }
-
-        if (_powers is { Length: > 0 })
-        {
-            NvApi.NvPowerTopology powerTopology = GetPowerTopology(out status);
-            if (status == NvApi.NvStatus.OK && powerTopology.Count > 0)
-            {
-                for (int i = 0; i < powerTopology.Count; i++)
                 {
-                    NvApi.NvPowerTopologyEntry entry = powerTopology.Entries[i];
+                    NvApi.NvUsages usages = GetUsages(out status);
-                    _powers[i].Value = entry.PowerUsage / 1000f;
+                    if (status == NvApi.NvStatus.OK)
+                    {
+                        for (int index = 0; index < usages.Entries.Length; index++)
+                        {
+                            NvApi.NvUsagesEntry load = usages.Entries[index];
+                            if (load.IsPresent > 0 && Enum.IsDefined(typeof(NvApi.NvUtilizationDomain), index))
+                                _loads[index].Value = load.Percentage;
+                        }
+                    }
                 }
             }
-        }
 
-        if (_coreVoltage != null)
+            if (_powers is { Length: > 0 })
-        {
-            var voltageSensor = GetVoltRailsStatus(out status);
-            if (status == NvApi.NvStatus.OK)
             {
-                var microvolts = ((ulong)voltageSensor.CoreMicrovoltsHigh << 32) | voltageSensor.CoreMicrovolts;
+                NvApi.NvPowerTopology powerTopology = GetPowerTopology(out status);
-
+                if (status == NvApi.NvStatus.OK && powerTopology.Count > 0)
-                _coreVoltage.Value = microvolts == 0 ? 0 : microvolts / 1_000_000f;
+                {
+                    for (int i = 0; i < powerTopology.Count; i++)
+                    {
+                        NvApi.NvPowerTopologyEntry entry = powerTopology.Entries[i];
+                        _powers[i].Value = entry.PowerUsage / 1000f;
+                    }
+                }
             }
-        }
 
-        if (NvApi.NvAPI_GPU_GetMemoryInfoEx != null || _displayHandle != null)
+            if (_coreVoltage != null)
-        {
-            uint free = 0;
-            uint total = 0;
-
-            //Size in bytes
-            NvApi.NvMemoryInfoEx memoryInfoEx = GetMemoryInfoEx(out status);
-            if (status == NvApi.NvStatus.OK)
             {
-                free = (uint)(memoryInfoEx.CurrentAvailableDedicatedVideoMemory / 1024);
+                var voltageSensor = GetVoltRailsStatus(out status);
-                total = (uint)(memoryInfoEx.DedicatedVideoMemory / 1024);
-            }
-            else
-            {
-                //Size in kilobytes, fallback for older drivers
-                NvApi.NvMemoryInfo memoryInfo = GetMemoryInfo(out status);
                 if (status == NvApi.NvStatus.OK)
                 {
-                    free = memoryInfo.CurrentAvailableDedicatedVideoMemory;
+                    var microvolts = ((ulong)voltageSensor.CoreMicrovoltsHigh << 32) | voltageSensor.CoreMicrovolts;
-                    total = memoryInfo.DedicatedVideoMemory;
+
+                    _coreVoltage.Value = microvolts == 0 ? 0 : microvolts / 1_000_000f;
                 }
             }
 
-            if (status == NvApi.NvStatus.OK)
+            if (NvApi.NvAPI_GPU_GetMemoryInfoEx != null || _displayHandle != null)
             {
-                _memoryTotal.Value = total / 1024;
+                uint free = 0;
-                ActivateSensor(_memoryTotal);
+                uint total = 0;
 
-                _memoryFree.Value = free / 1024;
+                //Size in bytes
-                ActivateSensor(_memoryFree);
+                NvApi.NvMemoryInfoEx memoryInfoEx = GetMemoryInfoEx(out status);
+                if (status == NvApi.NvStatus.OK)
+                {
+                    free = (uint)(memoryInfoEx.CurrentAvailableDedicatedVideoMemory / 1024);
+                    total = (uint)(memoryInfoEx.DedicatedVideoMemory / 1024);
+                }
+                else
+                {
+                    //Size in kilobytes, fallback for older drivers
+                    NvApi.NvMemoryInfo memoryInfo = GetMemoryInfo(out status);
+                    if (status == NvApi.NvStatus.OK)
+                    {
+                        free = memoryInfo.CurrentAvailableDedicatedVideoMemory;
+                        total = memoryInfo.DedicatedVideoMemory;
+                    }
+                }
 
-                _memoryUsed.Value = (total - free) / 1024;
+                if (status == NvApi.NvStatus.OK)
-                ActivateSensor(_memoryUsed);
+                {
+                    _memoryTotal.Value = total / 1024;
+                    ActivateSensor(_memoryTotal);
 
-                _memoryLoad.Value = ((float)(total - free) / total) * 100;
+                    _memoryFree.Value = free / 1024;
-                ActivateSensor(_memoryLoad);
+                    ActivateSensor(_memoryFree);
+
+                    _memoryUsed.Value = (total - free) / 1024;
+                    ActivateSensor(_memoryUsed);
+
+                    _memoryLoad.Value = ((float)(total - free) / total) * 100;
+                    ActivateSensor(_memoryLoad);
+                }
+            }
+
+            if (NvidiaML.IsAvailable && _nvmlDevice.HasValue)
+            {
+                int? result = NvidiaML.NvmlDeviceGetPowerUsage(_nvmlDevice.Value);
+                if (result.HasValue)
+                {
+                    _powerUsage.Value = result.Value / 1000f;
+                    ActivateSensor(_powerUsage);
+                }
+
+                // In MB/s, throughput sensors are passed as in KB/s.
+                uint? rx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.RxBytes);
+                if (rx.HasValue)
+                {
+                    _pcieThroughputRx.Value = rx * 1024;
+                    ActivateSensor(_pcieThroughputRx);
+                }
+
+                uint? tx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.TxBytes);
+                if (tx.HasValue)
+                {
+                    _pcieThroughputTx.Value = tx * 1024;
+                    ActivateSensor(_pcieThroughputTx);
+                }
+            }
+
+            // Astral specific
+            if (_12VHPwrPinCurrentSensors is { Length: > 0 } && TryReadAstral12VHPwrPinSensors(out ushort[] pinSensorValues))
+            {
+                float connectorCurrent = 0;
+                float connectorPower = 0;
+                for (int i = 0; i < 6; i++)
+                {
+                    float current = pinSensorValues[i * 2] / 1000f;
+                    float voltage = pinSensorValues[i * 2 + 1] / 1000f;
+                    _12VHPwrPinVoltageSensors[i].Value = voltage;
+                    _12VHPwrPinCurrentSensors[i].Value = current;
+                    _12VHPwrPinPowerSensors[i].Value = voltage * current;
+
+                    connectorCurrent += current;
+                    connectorPower += voltage * current;
+                }
+
+                _12VHPwrConnectorCurrentSensor.Value = connectorCurrent;
+                _12VHPwrConnectorPowerSensor.Value = connectorPower;
             }
         }
-
+        catch (Exception e)
-        if (NvidiaML.IsAvailable && _nvmlDevice.HasValue)
         {
-            int? result = NvidiaML.NvmlDeviceGetPowerUsage(_nvmlDevice.Value);
+            Debug.WriteLine($"{nameof(NvidiaGpu)} {nameof(Update)} failed for '{Name}' ({Identifier}): {e}");
-            if (result.HasValue)
-            {
-                _powerUsage.Value = result.Value / 1000f;
-                ActivateSensor(_powerUsage);
-            }
-
-            // In MB/s, throughput sensors are passed as in KB/s.
-            uint? rx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.RxBytes);
-            if (rx.HasValue)
-            {
-                _pcieThroughputRx.Value = rx * 1024;
-                ActivateSensor(_pcieThroughputRx);
-            }
-
-            uint? tx = NvidiaML.NvmlDeviceGetPcieThroughput(_nvmlDevice.Value, NvidiaML.NvmlPcieUtilCounter.TxBytes);
-            if (tx.HasValue)
-            {
-                _pcieThroughputTx.Value = tx * 1024;
-                ActivateSensor(_pcieThroughputTx);
-            }
-        }
-
-        // Astral specific
-        if (_12VHPwrPinCurrentSensors is { Length: > 0 } && TryReadAstral12VHPwrPinSensors(out ushort[] pinSensorValues))
-        {
-            float connectorCurrent = 0;
-            float connectorPower = 0;
-            for (int i = 0; i < 6; i++)
-            {
-                float current = pinSensorValues[i * 2] / 1000f;
-                float voltage = pinSensorValues[i * 2 + 1] / 1000f;
-                _12VHPwrPinVoltageSensors[i].Value = voltage;
-                _12VHPwrPinCurrentSensors[i].Value = current;
-                _12VHPwrPinPowerSensors[i].Value = voltage * current;
-
-                connectorCurrent += current;
-                connectorPower += voltage * current;
-            }
-
-            _12VHPwrConnectorCurrentSensor.Value = connectorCurrent;
-            _12VHPwrConnectorPowerSensor.Value = connectorPower;
         }
     }
 

LibreHardwareMonitorLib/Hardware/Gpu/NvidiaGroup.cs

@@ -4,100 +4,275 @@
 // Partial Copyright (C) Michael Möller <mmoeller@openhardwaremonitor.org> and Contributors.
 // All Rights Reserved.
 
+using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
 using LibreHardwareMonitor.Interop;
 
 namespace LibreHardwareMonitor.Hardware.Gpu;
 
-internal class NvidiaGroup : IGroup
+internal class NvidiaGroup : IGroup, IHardwareChanged
 {
-    private readonly List<Hardware> _hardware = new();
+    private readonly Dictionary<NvApi.NvPhysicalGpuHandle, NvidiaGpu> _hardwareByHandle = new();
+    private readonly List<Hardware> _hardware = [];
     private readonly StringBuilder _report = new();
+    private readonly ISettings _settings;
+
+    private CancellationTokenSource _cancellationTokenSource;
+    private bool _disposed;
+    private Task _monitorTask;
+    private bool _nvidiaWasAvailable;
 
     public NvidiaGroup(ISettings settings)
     {
-        NvApi.Initialize();
+        _settings = settings;
-
-        if (!NvApi.IsAvailable)
-            return;
 
         _report.AppendLine("NvApi");
         _report.AppendLine();
 
-        if (NvApi.NvAPI_GetInterfaceVersionString(out string version) == NvApi.NvStatus.OK)
+        if (Software.OperatingSystem.IsUnix)
         {
-            _report.Append("Version: ");
+            _report.AppendLine("Status: Not supported on Unix for NvApi group");
-            _report.AppendLine(version);
-        }
-
-        NvApi.NvPhysicalGpuHandle[] handles = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
-        if (NvApi.NvAPI_EnumPhysicalGPUs == null)
-        {
-            _report.AppendLine("Error: NvAPI_EnumPhysicalGPUs not available");
             _report.AppendLine();
             return;
         }
 
-        NvApi.NvStatus status = NvApi.NvAPI_EnumPhysicalGPUs(handles, out int count);
+        RefreshHardware(raiseEvents: false);
-        if (status != NvApi.NvStatus.OK)
+        StartMonitorTask();
-        {
-            _report.AppendLine("Status: " + status);
-            _report.AppendLine();
-            return;
-        }
-
-        IDictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> displayHandles = new Dictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle>();
-        if (NvApi.NvAPI_EnumNvidiaDisplayHandle != null && NvApi.NvAPI_GetPhysicalGPUsFromDisplay != null)
-        {
-            status = NvApi.NvStatus.OK;
-            int i = 0;
-            while (status == NvApi.NvStatus.OK)
-            {
-                NvApi.NvDisplayHandle displayHandle = new();
-                status = NvApi.NvAPI_EnumNvidiaDisplayHandle(i, ref displayHandle);
-                i++;
-
-                if (status == NvApi.NvStatus.OK)
-                {
-                    NvApi.NvPhysicalGpuHandle[] handlesFromDisplay = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
-                    if (NvApi.NvAPI_GetPhysicalGPUsFromDisplay(displayHandle, handlesFromDisplay, out uint countFromDisplay) == NvApi.NvStatus.OK)
-                    {
-                        for (int j = 0; j < countFromDisplay; j++)
-                        {
-                            if (!displayHandles.ContainsKey(handlesFromDisplay[j]))
-                                displayHandles.Add(handlesFromDisplay[j], displayHandle);
-                        }
-                    }
-                }
-            }
-        }
-
-        _report.Append("Number of GPUs: ");
-        _report.AppendLine(count.ToString(CultureInfo.InvariantCulture));
-
-        for (int i = 0; i < count; i++)
-        {
-            displayHandles.TryGetValue(handles[i], out NvApi.NvDisplayHandle displayHandle);
-            _hardware.Add(new NvidiaGpu(i, handles[i], displayHandle, settings));
-        }
-
-        _report.AppendLine();
     }
 
-    public IReadOnlyList<IHardware> Hardware => _hardware;
+    public event HardwareEventHandler HardwareAdded;
 
-    public string GetReport()
+    public event HardwareEventHandler HardwareRemoved;
+
+    public IReadOnlyList<IHardware> Hardware
     {
-        return _report.ToString();
+        get { return _hardware; }
     }
 
+    public string GetReport() => _report.ToString();
+
     public void Close()
     {
-        foreach (Hardware gpu in _hardware)
+        _disposed = true;
+
+        _cancellationTokenSource?.Cancel();
+
+        try
+        {
+            _monitorTask?.Wait(TimeSpan.FromSeconds(1));
+        }
+        catch (AggregateException)
+        {
+            // ignored
+        }
+
+        _cancellationTokenSource?.Dispose();
+        _cancellationTokenSource = null;
+        _monitorTask = null;
+
+        var hardwareToClose = _hardware.ToList();
+
+        _hardware.Clear();
+        _hardwareByHandle.Clear();
+
+        foreach (Hardware gpu in hardwareToClose)
             gpu.Close();
 
         NvidiaML.Close();
     }
+
+    private void StartMonitorTask()
+    {
+        CancellationTokenSource cts = new();
+        _cancellationTokenSource = cts;
+
+        CancellationToken token = cts.Token;
+
+        _monitorTask = Task.Run(async () =>
+                                {
+                                    while (!token.IsCancellationRequested)
+                                    {
+                                        try
+                                        {
+                                            await Task.Delay(TimeSpan.FromSeconds(1), token).ConfigureAwait(false);
+                                        }
+                                        catch (TaskCanceledException)
+                                        {
+                                            break;
+                                        }
+
+                                        if (_disposed)
+                                        {
+                                            break;
+                                        }
+
+                                        RefreshHardware(raiseEvents: true);
+                                    }
+                                },
+                                token);
+    }
+
+    private void RefreshHardware(bool raiseEvents)
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        bool isAvailable = TryEnumerateGpus(out NvApi.NvPhysicalGpuHandle[] handles, out int count);
+
+        if (!isAvailable)
+        {
+            if (_nvidiaWasAvailable)
+            {
+                NvidiaML.Close();
+            }
+
+            _nvidiaWasAvailable = false;
+            RemoveAllHardware(raiseEvents);
+
+            return;
+        }
+
+        // Driver was unavailable and came back: force NVML re-init
+        if (!_nvidiaWasAvailable)
+        {
+            NvidiaML.Close();
+
+            if (NvApi.NvAPI_GetInterfaceVersionString(out string version) == NvApi.NvStatus.OK)
+            {
+                _report.Append("Version: ");
+                _report.AppendLine(version);
+            }
+
+            _report.Append("Number of GPUs: ");
+            _report.AppendLine(count.ToString(CultureInfo.InvariantCulture));
+            _report.AppendLine();
+        }
+
+        _nvidiaWasAvailable = true;
+
+        IDictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> displayHandles = GetDisplayHandles();
+
+        HashSet<NvApi.NvPhysicalGpuHandle> currentSet = [.. handles.Take(count)];
+        List<Hardware> removed = [];
+
+        foreach (KeyValuePair<NvApi.NvPhysicalGpuHandle, NvidiaGpu> pair in _hardwareByHandle.ToList())
+        {
+            if (!currentSet.Contains(pair.Key))
+            {
+                _hardwareByHandle.Remove(pair.Key);
+                _hardware.Remove(pair.Value);
+                removed.Add(pair.Value);
+            }
+        }
+
+        for (int i = 0; i < count; ++i)
+        {
+            NvApi.NvPhysicalGpuHandle handle = handles[i];
+
+            if (_hardwareByHandle.ContainsKey(handle))
+                continue;
+
+            displayHandles.TryGetValue(handle, out NvApi.NvDisplayHandle displayHandle);
+
+            NvidiaGpu gpu = new(i, handle, displayHandle, _settings);
+            _hardwareByHandle.Add(handle, gpu);
+            _hardware.Add(gpu);
+
+            if (raiseEvents)
+                HardwareAdded?.Invoke(gpu);
+        }
+
+        foreach (Hardware gpu in removed)
+        {
+            if (raiseEvents)
+                HardwareRemoved?.Invoke(gpu);
+
+            gpu.Close();
+        }
+    }
+
+    private void RemoveAllHardware(bool raiseEvents)
+    {
+        if (_hardware.Count == 0)
+            return;
+
+        List<Hardware> removed = _hardware.ToList();
+
+        _hardware.Clear();
+        _hardwareByHandle.Clear();
+
+        foreach (Hardware gpu in removed)
+        {
+            if (raiseEvents)
+            {
+                HardwareRemoved?.Invoke(gpu);
+            }
+
+            gpu.Close();
+        }
+    }
+
+    private static IDictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> GetDisplayHandles()
+    {
+        Dictionary<NvApi.NvPhysicalGpuHandle, NvApi.NvDisplayHandle> displayHandles = [];
+
+        if (NvApi.NvAPI_EnumNvidiaDisplayHandle == null || NvApi.NvAPI_GetPhysicalGPUsFromDisplay == null)
+        {
+            return displayHandles;
+        }
+
+        NvApi.NvStatus status = NvApi.NvStatus.OK;
+        int i = 0;
+
+        while (status == NvApi.NvStatus.OK)
+        {
+            NvApi.NvDisplayHandle displayHandle = new();
+            status = NvApi.NvAPI_EnumNvidiaDisplayHandle(i, ref displayHandle);
+            i++;
+
+            if (status != NvApi.NvStatus.OK)
+            {
+                continue;
+            }
+
+            NvApi.NvPhysicalGpuHandle[] handlesFromDisplay = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
+            if (NvApi.NvAPI_GetPhysicalGPUsFromDisplay(displayHandle, handlesFromDisplay, out uint countFromDisplay) != NvApi.NvStatus.OK)
+            {
+                continue;
+            }
+
+            for (int j = 0; j < countFromDisplay; j++)
+            {
+                if (!displayHandles.ContainsKey(handlesFromDisplay[j]))
+                {
+                    displayHandles.Add(handlesFromDisplay[j], displayHandle);
+                }
+            }
+        }
+
+        return displayHandles;
+    }
+
+    private static bool TryEnumerateGpus(out NvApi.NvPhysicalGpuHandle[] handles, out int count)
+    {
+        handles = new NvApi.NvPhysicalGpuHandle[NvApi.MAX_PHYSICAL_GPUS];
+        count = 0;
+
+        NvApi.Initialize();
+
+        if (!NvApi.IsAvailable || NvApi.NvAPI_EnumPhysicalGPUs == null)
+        {
+            return false;
+        }
+
+        NvApi.NvStatus status = NvApi.NvAPI_EnumPhysicalGPUs(handles, out count);
+        return status == NvApi.NvStatus.OK && count > 0;
+    }
 }