From 83b882e6701549e2571b4cd8b2105a5bd696589b Mon Sep 17 00:00:00 2001
From: pavel <pavel.larkin@ikeja.co.za>
Date: Tue, 4 Nov 2025 16:46:04 +0200
Subject: [PATCH] 16:46

---
 files/ansible-playbooks/sot-updater.yml | 46 ++++++++++++++++---------
 1 file changed, 30 insertions(+), 16 deletions(-)

diff --git a/files/ansible-playbooks/sot-updater.yml b/files/ansible-playbooks/sot-updater.yml
index 621f2fe..239f12d 100644
--- a/files/ansible-playbooks/sot-updater.yml
+++ b/files/ansible-playbooks/sot-updater.yml
@@ -79,7 +79,6 @@
           - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
           - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
 
-
     # --- Parse the OK line robustly (token-based) ---
     - name: "NB preflight | Parse OK line (token)"
       when: nb_script.stat.exists
@@ -110,7 +109,6 @@
         nb_small: "{{ nb_kv.get('small', '') }}"
         nb_ok_line: "{{ nb_ok_line }}"
 
-   
     # --- Detect "cloud vs NetBox (before update) was different" (regex-free, robust)
     - name: "NB preflight | Detect whether IP changed (pre-update)"
       when: nb_script.stat.exists
@@ -126,21 +124,40 @@
         nb_ip_changed: "{{ reason != 'none' }}"
         nb_change_reason: "{{ reason }}"
 
+    # --- Detect cloud error (network/HTTP/500s) ---
+    - name: "NB preflight | Detect cloud error"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+      set_fact:
+        nb_cloud_error: "{{ (nb_preflight.rc | default(1)) != 0 or ('FAIL ' in out) }}"
+        nb_cloud_error_rc: "{{ nb_preflight.rc | default('n/a') }}"
+
     - name: "NB preflight | Verdict"
       when: nb_script.stat.exists
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
-          - "NB preflight verdict: {{ 'IP CHANGED (will requeue)' if nb_ip_changed else 'IP SAME (continue)' }}"
-          - "Reason: {{ nb_change_reason }}"
-   
-
-
+          - >-
+            NB preflight verdict:
+            {{ 'IP CHANGED (will requeue)' if (nb_ip_changed|default(false)) else
+               ('CLOUD ERROR (will requeue)' if (nb_cloud_error|default(false)) else
+               'OK (continue)') }}
+          - "Reason (IP): {{ nb_change_reason | default('none') }}"
+          - "rc={{ nb_cloud_error_rc | default('n/a') }}"
+          - >-
+            First FAIL line (if any):
+            {{
+              ((nb_preflight.stdout | default('')) | regex_replace('\r','')).
+              split('\n') | select('search','^FAIL ') | list | first | default('<none>')
+            }}
 
+    # --- If IP changed or cloud errored → publish a 3s delayed 'sot-updater' task and stop this host ---
     - name: "NB preflight | Publish delayed requeue (3s) and stop"
       when:
         - nb_script.stat.exists
-        - nb_ip_changed | default(false)
+        - (nb_ip_changed | default(false)) or (nb_cloud_error | default(false))
       delegate_to: localhost
       uri:
         url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
@@ -149,7 +166,7 @@
         password: "change_me"
         force_basic_auth: true
         status_code: 200
-        return_content: yes           # <-- make response body available
+        return_content: yes
         headers:
           content-type: "application/json"
         body_format: json
@@ -168,7 +185,7 @@
     - name: "NB preflight | Log requeue publish response"
       when:
         - nb_script.stat.exists
-        - nb_ip_changed | default(false)
+        - (nb_ip_changed | default(false)) or (nb_cloud_error | default(false))
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
@@ -180,19 +197,18 @@
     - name: "NB preflight | Stop further tasks for this host"
       when:
         - nb_script.stat.exists
-        - nb_ip_changed | default(false)
+        - (nb_ip_changed | default(false)) or (nb_cloud_error | default(false))
       meta: end_host
 
-    # --- If IP did not change → optional 1s pause, then continue normally ---
+    # --- If neither IP changed nor cloud error → optional 1s pause, then continue normally ---
     - name: "NB preflight | Pause 1s"
       when:
         - nb_script.stat.exists
-        - not (nb_ip_changed | default(false))
+        - not ((nb_ip_changed | default(false)) or (nb_cloud_error | default(false)))
       delegate_to: localhost
       ansible.builtin.pause:
         seconds: 1
 
-
   tasks:
     - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
       debug:
@@ -279,7 +295,6 @@
       failed_when: false
       ignore_errors: true
 
-
     - name: Refresh ARP #1
       raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
       failed_when: false
@@ -425,7 +440,6 @@
       failed_when: false
       ignore_errors: true
 
-
     # ----------------------- DEV2 firmux (simplified, literal) -----------------------
     - name: Dev2 | Read /usr/lib/release/firmux
       when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'