17:40

files/ansible-playbooks/afterupgrade_check.yml

@@ -23,6 +23,9 @@
     # Do NOT self-reference max_attempts. We’ll normalize below.
     max_attempts_default: 3
 
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
   tasks:
     # ---- Normalize metadata safely (no self-referential defaults) ----
     - name: Normalize metadata (no clever transforms)
@@ -35,6 +38,13 @@
         # preserve the original string verbatim for all subsequent retries
         target_version_full: "{{ target_version | default('') }}"
 
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
     - name: Show received metadata
       ansible.builtin.debug:
         msg:
@@ -116,7 +126,65 @@
     - name: Evaluate version match (full-string contains check)
       when: nc_probe.rc == 0 and banner_probe.rc == 0
       ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
 
     # ---- Journaling paths ----
     # Success: banner matches expected full target_version
@@ -129,8 +197,9 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
 
     - name: Publish success journal to control queue
       when: journal_success_payload is defined
@@ -195,7 +264,7 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
 
     - name: Publish mismatch journal to control queue

files/ansible-playbooks/afterupgrade_check_original.yml

@@ -0,0 +1,363 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_withcurl.yml

@@ -0,0 +1,437 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+# --- Manual run defaults (so we can execute without -e) --- 
+# These are safe to leave here; anything passed via -e will still override them.
+    attempt: 1 
+    max_attempts: 3 
+    current_delay_sec: 300 
+    correlation_id: "6f680073dc7c" 
+    original_emitted_at: "2025-10-30T18:46:52Z" 
+    target_version: "2.2.3 rev 9800" 
+    # Intentionally keep this empty to exercise the target_version_effective logic. 
+    target_version_full: "" 
+    schema_version: 1
+
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/sot-updater-upgradecmd.yml

@@ -0,0 +1,684 @@
+---
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing (path typo?)"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }} (chatty)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    # --- Debug-only: capture the exact upgrade_cmd line and stop the play ---
+    - name: "NB preflight | Extract exact upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\s*upgrade_cmd\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Show captured upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "upgrade_cmd line (exact): {{ nb_upgrade_line | default('<not found>') }}"
+          - "found? {{ (nb_upgrade_line | length) > 0 }}"
+
+    - name: "NB preflight | Parse value after '=' via awk"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args:
+        executable: /bin/bash
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set and show parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd value: '{{ nb_upgrade_cmd }}'"
+          - "len={{ nb_upgrade_cmd | length }}"
+
+    # --- Parse the OK line robustly (token-based) ---
+    - name: "NB preflight | Parse OK line (token)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=')
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc | default(1)) == 0 and (nb_ok_line | length) > 0 }}"
+        nb_dev: "{{ (nb_tokens | first | default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip', '') }}"
+        nb_fw: "{{ nb_kv.get('fw', '') }}"
+        nb_node: "{{ nb_kv.get('node', '') }}"
+        nb_sector: "{{ nb_kv.get('sector', '') }}"
+        nb_small: "{{ nb_kv.get('small', '') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    # --- Detect "cloud vs NetBox (before update) was different" (regex-free, robust)
+    - name: "NB preflight | Detect whether IP changed (pre-update)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight verdict: {{ 'IP CHANGED (will requeue)' if nb_ip_changed else 'IP SAME (continue)' }}"
+          - "Reason: {{ nb_change_reason }}"
+
+    # --- If IP changed → publish a 3s delayed 'sot-updater' and stop this host ---
+    - name: "NB preflight | Publish delayed requeue (3s) and stop"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers: { x-delay: 3000 }
+          routing_key: "deviceconfig"
+          payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater'} | to_json }}"
+          payload_encoding: "string"
+      register: rmq_requeue
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Log requeue publish response"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "RMQ publish URL: http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+          - "HTTP status: {{ rmq_requeue.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_requeue.json | default('<none>') }}"
+          - "Raw content: {{ rmq_requeue.content | default('<none>') }}"
+
+    - name: "NB preflight | Stop further tasks for this host"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+      meta: end_host
+
+    # --- If IP did not change → optional 1s pause, then continue normally ---
+    - name: "NB preflight | Pause 1s"
+      when:
+        - nb_script.stat.exists
+        - not (nb_ip_changed | default(false))
+      delegate_to: localhost
+      ansible.builtin.pause:
+        seconds: 1
+
+  tasks:
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug result of adding temp IP to DEV1
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_mac is defined and dev2_mac | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug ARP action summary on DEV1
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- ARP refresh ----------------------------
+    - name: Refresh ARP #1
+      ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Refresh ARP #1
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Local tunnel prep -----------------------
+    - name: Pick a free local TCP port for the tunnel
+      delegate_to: localhost
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Record chosen port and create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set facts for tunnel paths
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
+
+    # ----------------------- AIRPING #2 -----------------------
+    - name: Refresh ARP #2
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Start tunnel -----------------------
+    - name: Start SSH tunnel via DEV1
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- DEV1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
+
+    - name: Debug Dev1 normalized firmware
+      delegate_to: localhost
+      debug:
+        msg: "Dev1 fw_version  {{ dev1_fw_clean | default('N/A') }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- DEV2 auth -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok | default(false)
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set dev2_passfile_used
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
+
+    # ----------------------- AIRPING #3 -----------------------
+    - name: Refresh ARP #3
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- DEV2 firmux (simplified, literal) -----------------------
+    - name: Dev2 | Read /usr/lib/release/firmux
+      when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
+      delegate_to: localhost
+      shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      register: dev2_firmux
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
+      delegate_to: localhost
+      run_once: true
+      set_fact:
+        indoor_fw_norm: >-
+          {{
+            (
+              (
+                dev2_firmux.stdout | default('') | regex_replace('\r','')
+              ) | split('\n')
+              | map('trim')
+              | select('truthy')
+              | list
+            ) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
+              | list
+              | first
+              | default(
+                  ((dev2_firmux.stdout | default('') | regex_replace('\r',''))
+                  | split('\n')
+                  | map('trim')
+                  | select('truthy')
+                  | list
+                  | first
+                  | default(''))
+                )
+              | trim
+          }}
+
+    - name: Publish Dev2 indoor_fwver
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP #4
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster
+      delegate_to: localhost
+      shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir
+      delegate_to: localhost
+      file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    # --- Determine success of Dev1 & Dev2 reads (controller-side heuristics) ---
+    - name: Post | Derive success flags for Dev1/Dev2 reads
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev1_read_ok: "{{ (dev1_fw_clean | default('') | trim) | length > 0 }}"
+        dev2_read_ok: "{{ (dev2_firmux is defined) and ((dev2_firmux.rc | default(1)) == 0) and ((dev2_firmux.stdout | default('') | trim) | length > 0) }}"
+
+    - name: Post | Debug success flags
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev1_read_ok={{ dev1_read_ok | default(false) }}"
+          - "dev2_read_ok={{ dev2_read_ok | default(false) }}"
+          - "nb_upgrade_cmd='{{ (nb_upgrade_cmd | default('')) }}' (len={{ (nb_upgrade_cmd | default('')) | length }})"
+
+    # --- If both reads OK and upgrade_cmd looks valid, publish journal + schedule upgrade ---
+    - name: Post | Build journal payload for planned upgrade
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        upgrade_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            device is configured to be upgraded with {{ nb_upgrade_cmd }}. requesting the upgrade
+
+    - name: Post | Publish journal to controls
+      when: upgrade_journal_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ upgrade_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_upgrade
+      changed_when: (rmq_journal_upgrade.json is defined) and (rmq_journal_upgrade.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: Post | Schedule upgrade task via delayed exchange (10s)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers: { x-delay: 10000 }
+          routing_key: "deviceconfig"
+          payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': (nb_upgrade_cmd | default('')) } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_schedule_upgrade
+      changed_when: false
+      failed_when: false
+
+    # === NEW: Tag only when scheduling path ran (success gate same as scheduling) ===
+    - name: Post | Build tag payload (auto-indoor-upgrade)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+        - rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_add1: "auto-indoor-upgrade"
+
+    - name: Post | Publish add-tag to controls
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add
+      changed_when: (rmq_tag_add.json is defined) and (rmq_tag_add.json.routed | default(false) | bool)
+
+    - name: Post | Log upgrade scheduling response
+      when: rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "Upgrade scheduled to deviceconfig.delayed in 10s"
+          - "HTTP status: {{ rmq_schedule_upgrade.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_schedule_upgrade.json | default('<none>') }}"
+          - "Raw content: {{ rmq_schedule_upgrade.content | default('<none>') }}"
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE"

files/ansible-playbooks/sot-updater.yml

@@ -1,35 +1,34 @@
 ---
-# updater.yml — read Dev1 & Dev2 firmware versions and publish to controls → netbox-reporter
-# Run: nbplay updater.yml <device>
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
 
-- name: Read fw on Dev1 + Dev2, publish NetBox custom fields
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
   hosts: all
   gather_facts: no
 
   vars:
-    # Busybox-safe PATH prefix for raw calls on DEV1 (as in update-rebootin222)
     pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
-
-    # DEV1 credentials (as in update-indoor)
     dev1_user: "root"
     dev1_pass: "wavewave"
+    ssh_timeout: 30
 
-    # Tunnel target DEV2 behind DEV1 (as in update-indoor)
     dev2_host: "192.168.1.1"
     dev2_port: 22
     dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
     dev2_side_ip: "192.168.1.11/24"
     dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
 
-    # Password files sequence (as in update-indoor: try basicpass then basicpass2)
-    # NOTE: The selection below mirrors the exact “try #1 → select → try #2 → select → NONE” flow from update-indoor.
-    # Do not alter ordering.
-    # (Values are file names as used in your repo/environment.)
-    # We do not loop; we replicate the same task structure.
-    # The read of firmux uses whichever got selected.
-    # — Pavel’s rule: on-device commands remain immutable; we’re only orchestrating controller-side steps here.
-    
-    # Rabbit journaling (mirrors update-indoor / rebootin222)
     rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
     rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
     rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
@@ -38,25 +37,361 @@
     rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
 
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing (path typo?)"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }} (chatty)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    # --- Parse the OK line robustly (token-based) ---
+    - name: "NB preflight | Parse OK line (token)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=')
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc | default(1)) == 0 and (nb_ok_line | length) > 0 }}"
+        nb_dev: "{{ (nb_tokens | first | default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip', '') }}"
+        nb_fw: "{{ nb_kv.get('fw', '') }}"
+        nb_node: "{{ nb_kv.get('node', '') }}"
+        nb_sector: "{{ nb_kv.get('sector', '') }}"
+        nb_small: "{{ nb_kv.get('small', '') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    # --- Detect "cloud vs NetBox (before update) was different" (regex-free, robust)
+    - name: "NB preflight | Detect whether IP changed (pre-update)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    # --- Detect cloud error (network/HTTP/500s) ---
+    - name: "NB preflight | Detect cloud error"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+      set_fact:
+        nb_cloud_error: "{{ (nb_preflight.rc | default(1)) != 0 or ('FAIL ' in out) }}"
+        nb_cloud_error_rc: "{{ nb_preflight.rc | default('n/a') }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - >-
+            NB preflight verdict:
+            {{ 'IP CHANGED (will requeue)' if (nb_ip_changed|default(false)) else
+               ('CLOUD ERROR (will requeue)' if (nb_cloud_error|default(false)) else
+               'OK (continue)') }}
+          - "Reason (IP): {{ nb_change_reason | default('none') }}"
+          - "rc={{ nb_cloud_error_rc | default('n/a') }}"
+          - >-
+            First FAIL line (if any):
+            {{
+              ((nb_preflight.stdout | default('')) | regex_replace('\r','')).
+              split('\n') | select('search','^FAIL ') | list | first | default('<none>')
+            }}
+
+    # --- If IP changed or cloud errored → publish a 3s delayed 'sot-updater' task and stop this host ---
+    - name: "NB preflight | Publish delayed requeue (3s) and stop"
+      when:
+        - nb_script.stat.exists
+        - (nb_ip_changed | default(false)) or (nb_cloud_error | default(false))
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers:
+              x-delay: 3000
+          routing_key: "deviceconfig"
+          payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater'} | to_json }}"
+          payload_encoding: "string"
+      register: rmq_requeue
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Log requeue publish response"
+      when:
+        - nb_script.stat.exists
+        - (nb_ip_changed | default(false)) or (nb_cloud_error | default(false))
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "RMQ publish URL: http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+          - "HTTP status: {{ rmq_requeue.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_requeue.json | default('<none>') }}"
+          - "Raw content: {{ rmq_requeue.content | default('<none>') }}"
+
+    - name: "NB preflight | Stop further tasks for this host"
+      when:
+        - nb_script.stat.exists
+        - (nb_ip_changed | default(false)) or (nb_cloud_error | default(false))
+      meta: end_host
+
+    # --- If neither IP changed nor cloud error → optional 1s pause, then continue normally ---
+    - name: "NB preflight | Pause 1s"
+      when:
+        - nb_script.stat.exists
+        - not ((nb_ip_changed | default(false)) or (nb_cloud_error | default(false)))
+      delegate_to: localhost
+      ansible.builtin.pause:
+        seconds: 1
+
   tasks:
-    # ----------------------------- DEV1: read current banner (exactly like update-rebootin222) -----------------------------
-    - name: Dev1 | Read first /etc/banner line containing 'rev'
-      ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
-      register: dev1_banner
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug result of adding temp IP to DEV1
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
       changed_when: false
 
-    - name: Dev1 | Normalize '... rev NNNN' → '...-rNNNN'
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev1_fw_norm: >-
-          {{
-            (dev1_banner.stdout | default('') | trim)
-            | regex_replace('\\s*[Rr][Ee][Vv]\\.??\\s*([0-9]+)\\s*$', '-r\\1')
-          }}
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
 
-    - name: Publish → controls | custom_field_set fw_version (Dev1)
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_mac is defined and dev2_mac | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug ARP action summary on DEV1
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- ARP refresh ----------------------------
+    - name: Refresh ARP #1
+      ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Refresh ARP #1
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Local tunnel prep -----------------------
+    - name: Pick a free local TCP port for the tunnel
       delegate_to: localhost
-      ansible.builtin.uri:
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Record chosen port and create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set facts for tunnel paths
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
+
+    # ----------------------- AIRPING #2 -----------------------
+    - name: Refresh ARP #2
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Start tunnel -----------------------
+    - name: Start SSH tunnel via DEV1
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- DEV1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
+
+    - name: Debug Dev1 normalized firmware
+      delegate_to: localhost
+      debug:
+        msg: "Dev1 fw_version  {{ dev1_fw_clean | default('N/A') }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
         url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
         method: POST
         user: "{{ rmq_user }}"
@@ -72,212 +407,85 @@
             'inscope_device': (ansible_hostname | default(inventory_hostname)),
             'task_name': 'custom_field_set',
             'task_add1': 'fw_version',
-            'task_result': (dev1_fw_norm | default('unknown'))
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
           } | to_json }}"
           payload_encoding: "string"
       changed_when: false
 
-    # ----------------------------- TUNNEL PREP (update-indoor blocks copied) -----------------------------
-    - name: Add temporary IP on DEV1 (tolerate 'File exists')
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
-      register: add_ip
-      changed_when: add_ip.rc == 0
-      failed_when: add_ip.rc != 0
-                   and ('File exists' not in (add_ip.stdout | default('')))
-                   and ('File exists' not in (add_ip.stderr | default('')))
-
-    - name: Pick a free local TCP port for the tunnel (controller)
+    # ----------------------- DEV2 auth -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok | default(false)
       delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        pick() {
-            for i in $(seq 1 25); do
-                p="$(shuf -i 20000-39999 -n 1)"
-                if command -v ss >/dev/null 2>&1; then
-                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
-                        echo "$p"; return 0
-                    fi
-                else
-                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
-                        echo "$p"; return 0
-                    fi
-                fi
-            done
-            return 1
-        }
-        pick
-      register: pick_port
-      changed_when: false
-
-    - name: Stop if no free local port was found
-      ansible.builtin.meta: end_host
-      when: (pick_port.stdout | trim | length) == 0
-
-    - name: Record chosen local port
-      delegate_to: localhost
-      set_fact:
-        _local_port: "{{ pick_port.stdout | trim }}"
-
-    - name: Create ControlMaster socket dir (mktemp)
-      delegate_to: localhost
-      set_fact:
-        _ctrl_dir: "{{ lookup('pipe', 'mktemp -d') }}"
-
-    - name: Compose ControlMaster socket path
-      delegate_to: localhost
-      set_fact:
-        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
-
-
-    - name: Start SSH ControlMaster and forward 127.0.0.1:local → DEV2:22 via DEV1
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        USER="{{ dev1_user }}"
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        sshpass -p '{{ dev1_pass }}' ssh -f -N \
-            -o PreferredAuthentications=password -o PubkeyAuthentication=no \
-            -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30 \
-            -M -S "{{ _ctrl_sock }}" \
-            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
-            "${USER}@${HOST}"
-      args: { executable: /bin/bash }
-      register: start_tunnel
-      changed_when: true
-
-    - name: Probe TCP reachability to DEV2 through the tunnel
-      delegate_to: localhost
-      ansible.builtin.shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
-      register: nc_probe
-      changed_when: false
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
       ignore_errors: true
 
-    - name: Stop if tunnel TCP probe failed
-      ansible.builtin.meta: end_host
-      when: nc_probe.rc != 0
-
-    # ----------------------------- DEV2 AUTH PICK (exact task sequence from update-indoor) -----------------------------
-    - name: Try DEV2 login with 'basicpass' (root)
+    - name: Set dev2_passfile_used
       delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f basicpass ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=30 \
-          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
-      register: dev2_try_basicpass
-      changed_when: false
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
+
+    # ----------------------- AIRPING #3 -----------------------
+    - name: Refresh ARP #3
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
       ignore_errors: true
 
-    - name: Select 'basicpass' if previous login succeeded
-      when: dev2_try_basicpass.rc == 0
+    # ----------------------- DEV2 firmux (simplified, literal) -----------------------
+    - name: Dev2 | Read /usr/lib/release/firmux
+      when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
       delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "basicpass"
-      changed_when: false
-
-    - name: Try DEV2 login with 'basicpass2' (only if first failed)
-      when: dev2_passfile_used is not defined
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f basicpass2 ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=30 \
-          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
-      register: dev2_try_basicpass2
-      changed_when: false
-      ignore_errors: true
-
-    - name: Select 'basicpass2' if previous login succeeded
-      when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "basicpass2"
-      changed_when: false
-
-    - name: Mark DEV2 auth as NONE if both attempts failed
-      when: dev2_passfile_used is not defined
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "NONE"
-      changed_when: false
-
-    # ----------------------------- DEV2 firmware read (update-indoor firmux primary check) -----------------------------
-    - name: Dev2 | Read /usr/lib/release/firmux (if present)
-      when: dev2_passfile_used != "NONE"
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
+      shell: |
         PORT="{{ _local_port }}"
         sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=30 \
+          -o ConnectTimeout=15 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /usr/lib/release/firmux 2>/dev/null || true"
-      args: { executable: /bin/bash }
       register: dev2_firmux
-      changed_when: false
+      failed_when: false
       ignore_errors: true
 
-    - name: Dev2 | Fallback to banner 'rev' if firmux not available
-      when: dev2_passfile_used != "NONE" and ((dev2_firmux.stdout | default('') | trim) | length == 0)
+    - name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
       delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=30 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "grep -i rev /etc/banner 2>/dev/null | head -n1 || true"
-      args: { executable: /bin/bash }
-      register: dev2_banner
-      changed_when: false
-      ignore_errors: true
-
-    - name: Dev2 | Choose raw string (firmux preferred, else banner)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        indoor_fw_raw: >-
-          {{
-            (dev2_firmux.stdout | default('') | trim)
-            if ((dev2_firmux.stdout | default('') | trim) | length > 0)
-            else (dev2_banner.stdout | default('') | trim)
-          }}
-
-    - name: Dev2 | Normalize to X.Y.Z-rNNNN (convert trailing 'rev NNNN' → '-rNNNN')
-      delegate_to: localhost
-      ansible.builtin.set_fact:
+      run_once: true
+      set_fact:
         indoor_fw_norm: >-
           {{
-            ((indoor_fw_raw | default('') | trim | lower) is search('-r[0-9]+$'))
-            | ternary(
-                (indoor_fw_raw | default('') | trim),
-                ((indoor_fw_raw | default('') | trim) | regex_replace('\\s*[Rr][Ee][Vv]\\.??\\s*([0-9]+)\\s*$', '-r\\1'))
-              )
+            (
+              (
+                dev2_firmux.stdout | default('') | regex_replace('\r','')
+              ) | split('\n')
+              | map('trim')
+              | select('truthy')
+              | list
+            ) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
+              | list
+              | first
+              | default(
+                  ((dev2_firmux.stdout | default('') | regex_replace('\r','')) 
+                  | split('\n') 
+                  | map('trim') 
+                  | select('truthy') 
+                  | list 
+                  | first 
+                  | default(''))
+                )
+              | trim
           }}
 
-    - name: Publish → controls | custom_field_set indoor_fwver (Dev2)
-      when:
-        - dev2_passfile_used != "NONE"
-        - indoor_fw_norm is defined
-        - (indoor_fw_norm | length) > 0
+    - name: Publish Dev2 indoor_fwver
       delegate_to: localhost
-      ansible.builtin.uri:
+      uri:
         url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
         method: POST
         user: "{{ rmq_user }}"
@@ -293,31 +501,41 @@
             'inscope_device': (ansible_hostname | default(inventory_hostname)),
             'task_name': 'custom_field_set',
             'task_add1': 'indoor_fwver',
-            'task_result': indoor_fw_norm
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
           } | to_json }}"
           payload_encoding: "string"
       changed_when: false
 
   post_tasks:
-    - name: Close SSH ControlMaster (best-effort)
-      delegate_to: localhost
-      ansible.builtin.shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
-      changed_when: false
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP #4
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
       ignore_errors: true
 
-    - name: Remove tunnel control dir (best-effort)
+    - name: Close SSH ControlMaster
       delegate_to: localhost
-      ansible.builtin.file:
+      shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir
+      delegate_to: localhost
+      file:
         path: "{{ _ctrl_dir | default('/tmp/none') }}"
         state: absent
       ignore_errors: true
 
-    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
-      register: del_ip
-      changed_when: del_ip.rc == 0
-      failed_when: del_ip.rc != 0
-                   and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
-                   and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE"

files/ansible-playbooks/update-indoor.yml

@@ -1,4 +1,3 @@
----
 # update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
@@ -70,6 +69,31 @@
           - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
           - "inventory_hostname={{ inventory_hostname }}"
 
+    # ====== NEW: pick up force-upgrade flag from CLI and TEMP override ======
+    - name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | lower }}"
+
+    # >>> TEMPORARY OVERRIDE (REMOVE THIS TASK LATER) <<<
+    - name: TEMPORARILY set force-upgrade to yes (REMOVE BEFORE COMMITTING)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "yes"
+    # <<< END TEMPORARY >>>
+
+    - name: Normalize force-upgrade flag to boolean
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        force_upgrade: "{{ force_upgrade_raw in ['y','yes','1','true'] }}"
+
+    - name: Debug force-upgrade effective
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "force_upgrade_raw={{ force_upgrade_raw | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+
     - name: Stop early if connected DEV1 hostname != inventory
       ansible.builtin.meta: end_host
       when: (dev1_host_read.stdout | trim | length > 0) and
@@ -618,6 +642,57 @@
       ansible.builtin.debug:
         msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
 
+
+
+
+    # ===== NEW: derive expected & current firmware cores and block if equal (unless force-upgrade) =====
+    - name: Derive expected target_version core (early, from image filename)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_fw_core_early: "{{ (image_filename | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default('', true) }}"
+
+    - name: Parse current firmware core from DEV2 firmux (early, '-rNNNN' form)
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        current_fw_core_dash: "{{ (dev2_firmux.stdout | default('') | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default('', true) }}"
+
+    - name: Parse current firmware core from DEV2 firmux (alt 'X.Y.Z rev NNNN' → 'X.Y.Z-rNNNN')
+      when: dev2_firmux is defined and (current_fw_core_dash | length) == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        current_fw_core_rev_parts: "{{ (dev2_firmux.stdout | default('') | regex_search('([0-9]+\\.[0-9]+\\.[0-9]+)\\s+rev\\s+([0-9]+)')) | default('', true) }}"
+
+    - name: Normalize current firmware core (prefer '-r' form; else build from 'rev')
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        current_fw_core: >-
+          {% if (current_fw_core_dash | default('') | length) > 0 -%}
+          {{ current_fw_core_dash }}
+          {%- elif (current_fw_core_rev_parts | default('') | length) > 0 -%}
+          {{ (dev2_firmux.stdout | regex_replace('.*([0-9]+\\.[0-9]+\\.[0-9]+)\\s+rev\\s+([0-9]+).*','\\1-r\\2')) }}
+          {%- else -%}
+          {{ '' }}
+          {%- endif %}
+
+    - name: Debug version comparison (current vs expected; force-override)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
+          - "current_fw_core={{ current_fw_core | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+
+    - name: Soft-block if current firmware already matches target and not forced
+      when:
+        - not (force_upgrade | default(false))
+        - (expected_fw_core_early | length) > 0
+        - (current_fw_core | length) > 0
+        - current_fw_core == expected_fw_core_early
+      ansible.builtin.set_fact:
+        _blocked: true
+        _journal: "{{ (_journal | default([])) + [ 'Device already running target image: ' ~ current_fw_core ~ ' — skipping write/flip (use force-upgrade=yes to override)' ] }}"
+
     # ---------------------------- Normalize rebootin early (HOURS) ----------------------------
     # Strict: rebootin is integer HOURS only. Always add +20s grace to the schedule.
 
@@ -1334,6 +1409,9 @@
         msg:
           - "dev2_passfile_used={{ dev2_passfile_used }}"
           - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+          - "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
+          - "current_fw_core={{ current_fw_core | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
           - "local_image_present={{ local_img.stat.exists | default(false) }}"
           - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
           - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"

files/ansible-playbooks/update-reboot.yml

@@ -1,4 +1,4 @@
 ---
 # update-reboot.yml — thin wrapper, no var forwarding. 
 # Delegates entirely to the unified updater.
-- import_playbook: update-rebootin222.yml
+- import_playbook: update-rebootin223.yml

files/ansible-playbooks/update-rebootin223.yml

@@ -0,0 +1,809 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.3-r9800.bin
+    firmware_sha256: "5c06496a896831c5548988ac2575a0a84bb9054a769159dec354e1f996c252e4"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.3-r9800.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: "{{ au_delay_sec | default(300) }}"
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-upgrade payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterupgrade_payload={{ afterupgrade_payload | to_nice_json }}"
+              - "au_delay_sec={{ au_delay_sec }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost

files/nb_onedevice_update.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""
+nb_sync_one_device.py
+
+Update a single NetBox device from Cloud by hostname — ONLY if device is online.
+
+Behavior
+- Liveness gate via /v1/devices/{cloud_id}/liveness (no changes if offline).
+- Updates NetBox fields from Cloud:
+  custom_fields.fw_version ← firmwareVersion
+  custom_fields.nodeName   ← nodeName
+  custom_fields.sectorName ← sectorName
+  custom_fields.smallCellName ← smallCellName
+  serial ← serialNumber
+- IP handling:
+  If Cloud ipAddress is valid (not None/""/"0.0.0.0"):
+    ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
+    then PRUNE all other IPs on this device (default).
+  If Cloud ipAddress is placeholder/invalid:
+    skip IP changes and do not prune.
+- --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
+- NEW: Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+
+Exit codes:
+  0 = success
+  1 = not found / missing data / offline
+  3 = network/HTTP error
+  4 = NetBox update error
+"""
+
+import sys
+import json
+import argparse
+from typing import Optional, Union, List, Dict
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# ------------ HARD-CODED CONFIG (per Pavel) ------------
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v1/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
+REQ_TIMEOUT = 30
+CHATTY = False
+
+# ------------ HTTP utilities ------------
+def _new_session() -> requests.Session:
+    s = requests.Session()
+    retries = Retry(
+        total=3, connect=3, read=3, status=3,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+        respect_retry_after_header=True,
+    )
+    adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
+    s.mount('http://', adapter)
+    s.mount('https://', adapter)
+    s.headers['Accept'] = 'application/json'
+    return s
+
+S_NB = _new_session()
+S_CL = _new_session()
+
+# ------------ Logging / status helpers ------------
+def log(msg: str):
+    if CHATTY:
+        print(msg, file=sys.stderr)
+
+def die(code: int, msg: str):
+    # Single-line result: stdout on success, stderr on failure
+    print(msg, file=sys.stdout if code == 0 else sys.stderr)
+    raise SystemExit(code)
+
+# ------------ NetBox API ------------
+class NetBox:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip('/')
+        self.token = token
+
+    def _h(self):
+        return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def get_device_by_name(self, name: str) -> Optional[dict]:
+        log(f"NB: lookup device by name {name}")
+        r = S_NB.get(self._url("/api/dcim/devices/"), headers=self._h(),
+                     params={"name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET devices name={name} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def get_device(self, dev_id: int) -> dict:
+        log(f"NB: fetch device id={dev_id}")
+        r = S_NB.get(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET device id={dev_id} HTTP={r.status_code}")
+        return r.json()
+
+    def patch_device(self, dev_id: int, patch: dict) -> None:
+        if not patch:
+            log("NB: no device patch needed")
+            return
+        log(f"NB: patch device id={dev_id} keys={list(patch.keys())}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps(patch), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL patch device dev={dev_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                     params={"device_id": dev_id, "name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET interfaces device_id={dev_id} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0]["id"] if res else None
+
+    def ensure_eth0(self, dev_id: int) -> int:
+        ifid = self.get_iface_id(dev_id, "eth0")
+        log(f"NB: ensure eth0 (current id={ifid})")
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = S_NB.post(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        if r.status_code == 400:
+            # race: read again
+            ifid = self.get_iface_id(dev_id, "eth0")
+            if ifid:
+                return ifid
+        die(4, f"FAIL create eth0 HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_ip_by_addr(self, addr: str) -> Optional[dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"address": f"{addr}/32"}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET ip-addresses addr={addr} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def create_ip_for_iface(self, addr: str, iface_id: int) -> int:
+        payload = {
+            "address": f"{addr}/32",
+            "status": "active",
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        }
+        r = S_NB.post(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        die(4, f"FAIL create IP {addr} HTTP={r.status_code} body={r.text[:200]}")
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
+        r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
+                       data=json.dumps({
+                           "assigned_object_type": "dcim.interface",
+                           "assigned_object_id": iface_id
+                       }),
+                       timeout=REQ_TIMEOUT)
+        return 200 <= r.status_code < 300
+
+    def device_set_primary_ip4(self, dev_id: int, ip_id: int) -> None:
+        log(f"NB: set primary_ip4 dev={dev_id} -> ip_id={ip_id}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps({"primary_ip4": ip_id}), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL set primary_ip4 dev={dev_id} ip_id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = S_NB.get(self._url(f"/api/dcim/interfaces/{iface_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return (r.json().get("device") or {}).get("id")
+        return None
+
+    # --- Helpers for pruning ---
+    def list_device_ips(self, dev_id: int) -> List[Dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return r.json().get("results") or []
+        ips: List[Dict] = []
+        r2 = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r2.status_code == 200:
+            for iface in (r2.json().get("results") or []):
+                ifid = iface.get("id")
+                r3 = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                              params={"assigned_object_type": "dcim.interface",
+                                      "assigned_object_id": ifid, "limit": 1000}, timeout=REQ_TIMEOUT)
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get("results") or [])
+        return ips
+
+    def delete_ip(self, ip_id: int) -> None:
+        log(f"NB: delete IP id={ip_id}")
+        r = S_NB.delete(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300 or r.status_code == 204):
+            die(4, f"FAIL delete IP id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+# ------------ Cloud API ------------
+class Cloud:
+    def __init__(self, base: str, bearer: str):
+        self.base = base.rstrip("/")
+        self.bearer = bearer
+
+    def _h(self):
+        return {"Authorization": f"Bearer {self.bearer}", "Accept": "application/json"}
+
+    def device_detail(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}"
+        log(f"CL: fetch detail cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id} HTTP={r.status_code}")
+        return r.json()
+
+    def device_liveness(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}/liveness"
+        log(f"CL: fetch liveness cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
+        return r.json()
+
+# ------------ Core ------------
+def _primary_ip4_text(dev_json: dict) -> Optional[str]:
+    p = dev_json.get("primary_ip4") or {}
+    addr = p.get("address")
+    if isinstance(addr, str) and addr.endswith("/32"):
+        return addr[:-3]
+    return addr
+
+def run(hostname: str) -> None:
+    nb = NetBox(NB_URL, NB_TOKEN)
+    cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+
+    dev = nb.get_device_by_name(hostname)
+    if not dev:
+        die(1, f"FAIL {hostname} not found in NetBox")
+
+    dev_id = dev["id"]
+    dev_full = nb.get_device(dev_id)
+    cf = dev_full.get("custom_fields") or {}
+
+    # NEW: log upgrade command if present
+    upgrade_cmd = cf.get("upgrade_cmd")
+    log(f"NB: upgrade_cmd={upgrade_cmd}")
+
+    cloud_id = cf.get("cloud_id")
+    if cloud_id in (None, "", "null"):
+        die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
+
+    # 1) Liveness gate
+    live = cl.device_liveness(cloud_id)
+    if not bool(live.get("isConnected")):
+        die(1, f"FAIL {hostname} cloud_id={cloud_id} device is not online")
+
+    current_nb_ip = _primary_ip4_text(dev_full)
+
+    # 2) Cloud detail
+    d = cl.device_detail(cloud_id)
+    fw = d.get("firmwareVersion") or d.get("version")
+    ip_from_cloud = (d.get("ipAddress") or "").strip() if isinstance(d.get("ipAddress"), str) else d.get("ipAddress")
+    node = d.get("nodeName")
+    sector = d.get("sectorName")
+    small = d.get("smallCellName")
+    serial = d.get("serialNumber")
+
+    if not fw:
+        die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
+
+    # 3) NetBox patch (idempotent)
+    cf_patch = {}
+    if cf.get("fw_version") != fw:
+        cf_patch["fw_version"] = fw
+        log(f"CF: fw_version -> {fw}")
+    if node and cf.get("nodeName") != node:
+        cf_patch["nodeName"] = node
+        log(f"CF: nodeName -> {node}")
+    if sector and cf.get("sectorName") != sector:
+        cf_patch["sectorName"] = sector
+        log(f"CF: sectorName -> {sector}")
+    if small and cf.get("smallCellName") != small:
+        cf_patch["smallCellName"] = small
+        log(f"CF: smallCellName -> {small}")
+
+    dev_patch = {}
+    if serial and (dev_full.get("serial") != serial):
+        dev_patch["serial"] = serial
+        log(f"DEV: serial -> {serial}")
+    if cf_patch:
+        dev_patch["custom_fields"] = cf_patch
+    nb.patch_device(dev_id, dev_patch)
+
+    # 4) IP handling (skip if placeholder)
+    ip_is_placeholder = (ip_from_cloud in (None, "", "0.0.0.0"))
+    ip_out_for_status = current_nb_ip  # default to current NB IP
+
+    if ip_is_placeholder:
+        log(f"IP: cloud reported placeholder '{ip_from_cloud}', skipping IP changes; keeping NetBox ip(s) as-is")
+    else:
+        iface_id = nb.ensure_eth0(dev_id)
+
+        ip_rec = nb.get_ip_by_addr(ip_from_cloud)
+        if ip_rec is None:
+            log(f"IP: create new {ip_from_cloud} on iface {iface_id}")
+            ip_id = nb.create_ip_for_iface(ip_from_cloud, iface_id)
+        else:
+            ip_id = ip_rec["id"]
+            assigned_type = ip_rec.get("assigned_object_type") or ""
+            assigned_id = ip_rec.get("assigned_object_id")
+            if not assigned_type:
+                if not nb.assign_ip_to_iface(ip_id, iface_id):
+                    die(4, f"FAIL assign IP {ip_from_cloud} to iface {iface_id}")
+            elif assigned_type == "dcim.interface":
+                if str(assigned_id) != str(iface_id):
+                    other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
+                    log(f"IP: moving {ip_from_cloud} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id}")
+                    # Clear old device primary if necessary
+                    if other_dev:
+                        r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
+                        if r.status_code == 200:
+                            old_primary_id = (r.json().get("primary_ip4") or {}).get("id")
+                            if str(old_primary_id) == str(ip_id):
+                                S_NB.patch(nb._url(f"/api/dcim/devices/{other_dev}/"),
+                                           headers=nb._h(), data=json.dumps({"primary_ip4": None}),
+                                           timeout=REQ_TIMEOUT)
+                    if not nb.assign_ip_to_iface(ip_id, iface_id):
+                        die(4, f"FAIL move IP {ip_from_cloud} to iface {iface_id}")
+            else:
+                die(4, f"FAIL IP {ip_from_cloud} assigned to {assigned_type}")
+
+        nb.device_set_primary_ip4(dev_id, ip_id)
+        ip_out_for_status = ip_from_cloud
+
+        # PRUNE all other IPs on this device (default behavior)
+        all_ips = nb.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get("id")
+            if str(rid) == str(ip_id):
+                continue
+            addr = rec.get("address")
+            log(f"IP: pruning stale {addr} (id={rid}) from device {dev_id}")
+            nb.delete_ip(rid)
+
+    print(f"OK {hostname} ip={ip_out_for_status or 'NONE'} fw={fw} node={node} sector={sector} small={small}")
+
+# ------------ CLI ------------
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname (only if online)")
+    ap.add_argument("hostname", help="Device name in NetBox")
+    ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
+    args = ap.parse_args()
+
+    CHATTY = bool(args.chatty)  # module-scope assignment
+    try:
+        run(args.hostname)
+    except requests.RequestException as e:
+        die(3, f"FAIL network error: {e}")