From 068f00141235f414827c8934349f83c3d815565b Mon Sep 17 00:00:00 2001
From: patrick <patrickjaroszewski@protonmail.com>
Date: Mon, 27 Jul 2026 16:54:41 -0400
Subject: [PATCH] fix(nvidia_drivers): scope cleanup to real verify failures,
 spare version mismatch

Replace the block/rescue that caught any error -- including templating errors in
the debug task -- with explicit conditional cleanup gated on nvidia-smi's actual
return code. A cosmetic failure can no longer roll back a working install. rc 18
(driver/library version mismatch) now fails with guidance and leaves packages intact.
---
 roles/nvidia_drivers/tasks/main.yml | 72 ++++++++++++++++++++---------
 1 file changed, 50 insertions(+), 22 deletions(-)

diff --git a/roles/nvidia_drivers/tasks/main.yml b/roles/nvidia_drivers/tasks/main.yml
index 52387ae..03d2046 100644
--- a/roles/nvidia_drivers/tasks/main.yml
+++ b/roles/nvidia_drivers/tasks/main.yml
@@ -75,28 +75,42 @@
     state: present
   when: nvidia_optional_packages | length > 0
 
-- name: Verify and handle NVIDIA driver installation
+# Capture the verification result without failing the play here, so that only a
+# genuine driver failure (evaluated explicitly below) can trigger the destructive
+# package cleanup. A cosmetic/templating error in a later task can no longer roll
+# back a working install.
+- name: Verify NVIDIA driver installation
+  ansible.builtin.command:
+    cmd: nvidia-smi
+  register: nvidia_verification
+  changed_when: false
+  failed_when: false
+  when: kernel_driver_install is changed or cuda_driver_install is changed
+
+# rc 18 = "Driver/library version mismatch": the driver is installed correctly but
+# the running kernel module differs from userspace. This needs a reboot (kernel-module
+# installs) or aligning the container driver to the Proxmox host version (LXC) -- it is
+# NOT a broken install, so leave the packages in place.
+- name: Fail on driver/library version mismatch (packages left intact)
+  ansible.builtin.fail:
+    msg: >-
+      nvidia-smi reports a driver/library version mismatch (rc 18). The driver is
+      installed but the running kernel module version differs from userspace. Reboot
+      the host for kernel-module installs, or align this container's NVIDIA userspace
+      driver to the Proxmox host's kernel driver version (LXC), then re-run. Packages
+      were left in place.
+  when:
+    - nvidia_verification.rc is defined
+    - nvidia_verification.rc == 18
+
+# Only a real verification failure (nvidia-smi ran and returned non-zero, excluding the
+# version-mismatch case above) rolls back the install.
+- name: Clean up failed NVIDIA driver installation
+  when:
+    - nvidia_verification.rc is defined
+    - nvidia_verification.rc != 0
+    - nvidia_verification.rc != 18
   block:
-    - name: Verify NVIDIA driver installation
-      ansible.builtin.command:
-        cmd: nvidia-smi
-      register: nvidia_verification
-      changed_when: false
-      when: kernel_driver_install is changed or cuda_driver_install is changed
-
-    - name: Display NVIDIA driver information
-      ansible.builtin.debug:
-        msg: >-
-          {{
-            nvidia_verification.stdout_lines
-            if (nvidia_verification.rc is defined and nvidia_verification.rc == 0)
-            else nvidia_check.stdout_lines
-          }}
-      when: >-
-        (nvidia_verification.rc is defined and nvidia_verification.rc == 0)
-        or nvidia_check.rc == 0
-
-  rescue:
     - name: Remove partially installed packages on failure
       ansible.builtin.apt:
         name:
@@ -113,4 +127,18 @@
 
     - name: Fail with helpful message
       ansible.builtin.fail:
-        msg: "NVIDIA driver installation failed. Packages have been cleaned up."
\ No newline at end of file
+        msg: >-
+          NVIDIA driver verification failed (nvidia-smi rc={{ nvidia_verification.rc }}).
+          Packages have been cleaned up. Output: {{ nvidia_verification.stdout | default('') }}
+
+- name: Display NVIDIA driver information
+  ansible.builtin.debug:
+    msg: >-
+      {{
+        nvidia_verification.stdout_lines
+        if (nvidia_verification.rc is defined and nvidia_verification.rc == 0)
+        else nvidia_check.stdout_lines
+      }}
+  when: >-
+    (nvidia_verification.rc is defined and nvidia_verification.rc == 0)
+    or nvidia_check.rc == 0
\ No newline at end of file