1713

files/ansible-playbooks/afterupgrade_check.yml

@@ -23,6 +23,9 @@
     # Do NOT self-reference max_attempts. We’ll normalize below.
     max_attempts_default: 3
 
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
   tasks:
     # ---- Normalize metadata safely (no self-referential defaults) ----
     - name: Normalize metadata (no clever transforms)
@@ -35,6 +38,13 @@
         # preserve the original string verbatim for all subsequent retries
         target_version_full: "{{ target_version | default('') }}"
 
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
     - name: Show received metadata
       ansible.builtin.debug:
         msg:
@@ -116,7 +126,65 @@
     - name: Evaluate version match (full-string contains check)
       when: nc_probe.rc == 0 and banner_probe.rc == 0
       ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
 
     # ---- Journaling paths ----
     # Success: banner matches expected full target_version
@@ -129,8 +197,9 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
 
     - name: Publish success journal to control queue
       when: journal_success_payload is defined
@@ -195,7 +264,7 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
 
     - name: Publish mismatch journal to control queue

files/ansible-playbooks/afterupgrade_check_original.yml

@@ -0,0 +1,363 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_withcurl.yml

@@ -0,0 +1,437 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+# --- Manual run defaults (so we can execute without -e) --- 
+# These are safe to leave here; anything passed via -e will still override them.
+    attempt: 1 
+    max_attempts: 3 
+    current_delay_sec: 300 
+    correlation_id: "6f680073dc7c" 
+    original_emitted_at: "2025-10-30T18:46:52Z" 
+    target_version: "2.2.3 rev 9800" 
+    # Intentionally keep this empty to exercise the target_version_effective logic. 
+    target_version_full: "" 
+    schema_version: 1
+
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/restart-cloud-agent.yml

@@ -0,0 +1,25 @@
+---
+- name: Temporarily move cloud-agent and restore it
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Move /tmp/launchd/services/cloud-agent to /root/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /tmp/launchd/services/cloud-agent /root/
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 3
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /root/cloud-agent /tmp/launchd/services/
+      register: move_back
+      changed_when: true

files/ansible-playbooks/sot-updater-upgradecmd.yml

@@ -0,0 +1,752 @@
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # === NEW (surgical): requeue attempt counter from environment, default 0 ===
+    requeue_attempt: "{{ (lookup('env','REQUEUE_ATTEMPT') | default('0', true)) | int }}"
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing (path typo?)"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }} (chatty)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    # === HARD STOP ON CLOUD FAILURE (immediately after preflight) ===
+    - name: "NB preflight | Detect cloud failure (rc==3 OR 'FAIL Cloud' line)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        _lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r',''))
+             | split('\n') | map('trim') | list }}
+        _cloud_fail_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
+        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
+      ansible.builtin.set_fact:
+        cloud_fail_line: "{{ _cloud_fail_line }}"
+        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"
+
+    - name: "NB preflight | Journal + STOP (cloud unavailable)"
+      when:
+        - nb_script.stat.exists
+        - cloud_bad | default(false)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'preflight abort: cloud unavailable; closing without requeue. '
+              ~ (cloud_fail_line | default(''))
+              ~ ' rc=' ~ ((nb_preflight.rc | default('')) | string)
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cloud_unavail
+      changed_when: (rmq_cloud_unavail.json is defined) and (rmq_cloud_unavail.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: "NB preflight | Stop host due to cloud unavailability (no requeue)"
+      when:
+        - nb_script.stat.exists
+        - cloud_bad | default(false)
+      ansible.builtin.meta: end_host
+    # === END HARD STOP ===
+
+    # --- Debug-only: capture the exact upgrade_cmd line and stop the play ---
+    - name: "NB preflight | Extract exact upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\s*upgrade_cmd\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Show captured upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "upgrade_cmd line (exact): {{ nb_upgrade_line | default('<not found>') }}"
+          - "found? {{ (nb_upgrade_line | length) > 0 }}"
+
+    - name: "NB preflight | Parse value after '=' via awk"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args:
+        executable: /bin/bash
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set and show parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd value: '{{ nb_upgrade_cmd }}'"
+          - "len={{ nb_upgrade_cmd | length }}"
+
+    # --- Parse the OK line robustly (token-based) ---
+    - name: "NB preflight | Parse OK line (token)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=')
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc | default(1)) == 0 and (nb_ok_line | length) > 0 }}"
+        nb_dev: "{{ (nb_tokens | first | default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip', '') }}"
+        nb_fw: "{{ nb_kv.get('fw', '') }}"
+        nb_node: "{{ nb_kv.get('node', '') }}"
+        nb_sector: "{{ nb_kv.get('sector', '') }}"
+        nb_small: "{{ nb_kv.get('small', '') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    # --- Detect "cloud vs NetBox (before update) was different" (regex-free, robust)
+    - name: "NB preflight | Detect whether IP changed (pre-update)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight verdict: {{ 'IP CHANGED (will requeue)' if nb_ip_changed else 'IP SAME (continue)' }}"
+          - "Reason: {{ nb_change_reason }}"
+
+    # --- If IP changed → publish a 3s delayed 'sot-updater' and stop this host ---
+    - name: "NB preflight | Publish delayed requeue (3s) and stop"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+        - (requeue_attempt | int) < 3              # <— NEW: limit to 3 tries
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            # === NEW (surgical): 20s delay instead of 3s ===
+            headers: { x-delay: 20000 }
+          routing_key: "deviceconfig"
+          # === NEW (surgical): include attempt counter in payload ===
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'sot-updater',
+            'attempt': ((requeue_attempt | int) + 1)
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_requeue
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Log requeue publish response"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+        - (requeue_attempt | int) < 3              # <— NEW: only if we actually published
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "RMQ publish URL: http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+          - "HTTP status: {{ rmq_requeue.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_requeue.json | default('<none>') }}"
+          - "Raw content: {{ rmq_requeue.content | default('<none>') }}"
+
+    - name: "NB preflight | Stop further tasks for this host"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+      meta: end_host
+
+    # --- If IP did not change → optional 1s pause, then continue normally ---
+    - name: "NB preflight | Pause 1s"
+      when:
+        - nb_script.stat.exists
+        - not (nb_ip_changed | default(false))
+      delegate_to: localhost
+      ansible.builtin.pause:
+        seconds: 1
+
+  tasks:
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug result of adding temp IP to DEV1
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_mac is defined and dev2_mac | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug ARP action summary on DEV1
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- ARP refresh ----------------------------
+    - name: Refresh ARP #1
+      ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Refresh ARP #1
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Local tunnel prep -----------------------
+    - name: Pick a free local TCP port for the tunnel
+      delegate_to: localhost
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Record chosen port and create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set facts for tunnel paths
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
+
+    # ----------------------- AIRPING #2 -----------------------
+    - name: Refresh ARP #2
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Start tunnel -----------------------
+    - name: Start SSH tunnel via DEV1
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- DEV1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
+
+    - name: Debug Dev1 normalized firmware
+      delegate_to: localhost
+      debug:
+        msg: "Dev1 fw_version  {{ dev1_fw_clean | default('N/A') }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- DEV2 auth -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok | default(false)
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set dev2_passfile_used
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
+
+    # ----------------------- AIRPING #3 -----------------------
+    - name: Refresh ARP #3
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- DEV2 firmux (simplified, literal) -----------------------
+    - name: Dev2 | Read /usr/lib/release/firmux
+      when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
+      delegate_to: localhost
+      shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      register: dev2_firmux
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
+      delegate_to: localhost
+      run_once: true
+      set_fact:
+        indoor_fw_norm: >-
+          {{
+            (
+              (
+                dev2_firmux.stdout | default('') | regex_replace('\r','')
+              ) | split('\n')
+              | map('trim')
+              | select('truthy')
+              | list
+            ) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
+              | list
+              | first
+              | default(
+                  ((dev2_firmux.stdout | default('') | regex_replace('\r',''))
+                  | split('\n')
+                  | map('trim')
+                  | select('truthy')
+                  | list
+                  | first
+                  | default(''))
+                )
+              | trim
+          }}
+
+    - name: Publish Dev2 indoor_fwver
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP #4
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster
+      delegate_to: localhost
+      shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir
+      delegate_to: localhost
+      file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    # --- Determine success of Dev1 & Dev2 reads (controller-side heuristics) ---
+    - name: Post | Derive success flags for Dev1/Dev2 reads
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev1_read_ok: "{{ (dev1_fw_clean | default('') | trim) | length > 0 }}"
+        dev2_read_ok: "{{ (dev2_firmux is defined) and ((dev2_firmux.rc | default(1)) == 0) and ((dev2_firmux.stdout | default('') | trim) | length > 0) }}"
+
+    - name: Post | Debug success flags
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev1_read_ok={{ dev1_read_ok | default(false) }}"
+          - "dev2_read_ok={{ dev2_read_ok | default(false) }}"
+          - "nb_upgrade_cmd='{{ (nb_upgrade_cmd | default('')) }}' (len={{ (nb_upgrade_cmd | default('')) | length }})"
+
+    # --- If both reads OK and upgrade_cmd looks valid, publish journal + schedule upgrade ---
+    - name: Post | Build journal payload for planned upgrade
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        upgrade_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            device is configured to be upgraded with {{ nb_upgrade_cmd }}. requesting the upgrade
+
+    - name: Post | Publish journal to controls
+      when: upgrade_journal_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ upgrade_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_upgrade
+      changed_when: (rmq_journal_upgrade.json is defined) and (rmq_journal_upgrade.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: Post | Schedule upgrade task via delayed exchange (10s)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers: { x-delay: 300000 }
+          routing_key: "deviceconfig"
+          payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': (nb_upgrade_cmd | default('')) } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_schedule_upgrade
+      changed_when: false
+      failed_when: false
+
+    # === NEW: Tag only when scheduling path ran (success gate same as scheduling) ===
+    - name: Post | Build tag payload (auto-indoor-upgrade)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+        - rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_add1: "auto-indoor-upgrade"
+
+    - name: Post | Publish add-tag to controls
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add
+      changed_when: (rmq_tag_add.json is defined) and (rmq_tag_add.json.routed | default(false) | bool)
+
+    - name: Post | Log upgrade scheduling response
+      when: rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "Upgrade scheduled to deviceconfig.delayed in 10s"
+          - "HTTP status: {{ rmq_schedule_upgrade.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_schedule_upgrade.json | default('<none>') }}"
+          - "Raw content: {{ rmq_schedule_upgrade.content | default('<none>') }}"
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE"

files/ansible-playbooks/sot-updater.yml

@@ -0,0 +1,807 @@
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode, no upgrade scheduling)
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry, no upgrade scheduling)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # --- Retry attempt tracking (default 0 if missing) ---
+    requeue_attempt: "{{ (lookup('env','REQUEUE_ATTEMPT') | default('0', true)) | int }}"
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }}"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    # --- Detect cloud failure (rc==3 or FAIL Cloud line) ---
+    - name: "NB preflight | Detect cloud failure"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        _lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        _cloud_fail_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
+        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
+      set_fact:
+        cloud_fail_line: "{{ _cloud_fail_line }}"
+        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"
+
+    # --- Extract upgrade_cmd line if any ---
+    - name: "NB preflight | Extract upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\\s*upgrade_cmd\\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Parse upgrade_cmd value"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args: { executable: /bin/bash }
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd='{{ nb_upgrade_cmd }}' (len={{ nb_upgrade_cmd|length }})"
+
+    # --- Parse OK line & detect IP change ---
+    - name: "NB preflight | Parse OK line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=')
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc|default(1)) == 0 and (nb_ok_line|length)>0 }}"
+        nb_dev: "{{ (nb_tokens|first|default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip','') }}"
+        nb_fw: "{{ nb_kv.get('fw','') }}"
+        nb_node: "{{ nb_kv.get('node','') }}"
+        nb_sector: "{{ nb_kv.get('sector','') }}"
+        nb_small: "{{ nb_kv.get('small','') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    - name: "NB preflight | Detect IP change"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      debug:
+        msg:
+          - "Preflight verdict:"
+          - "Cloud fail: {{ cloud_bad|default(false) }}"
+          - "IP changed: {{ nb_ip_changed|default(false) }}"
+          - "Reason: {{ nb_change_reason|default('none') }}"
+
+    - name: "NB preflight | Compute next_attempt"
+      delegate_to: localhost
+      set_fact:
+        next_attempt: "{{ (requeue_attempt | int) + 1 }}"
+
+    # --- Requeue control: only 3 attempts, 300s apart ---
+    - name: "NB preflight | Publish delayed requeue (300s (10s)) if needed"
+      when:
+        - nb_script.stat.exists
+        - (cloud_bad | default(false)) or (nb_ip_changed | default(false))
+        - (requeue_attempt | int) < 3
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers: { x-delay: 10000 }
+          routing_key: "deviceconfig"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'sot-updater',
+            'attempt': next_attempt
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_requeue
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Log requeue publish response"
+      when:
+        - nb_script.stat.exists
+        - ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
+        - (requeue_attempt | int) < 3
+      delegate_to: localhost
+      debug:
+        msg:
+          - "Requeue published (attempt {{ (requeue_attempt | int) + 1 }})"
+          - "HTTP: {{ rmq_requeue.status|default('n/a') }}"
+          - "JSON: {{ rmq_requeue.json|default('<none>') }}"
+
+    - name: "NB preflight | Stop host after requeue"
+      when:
+        - nb_script.stat.exists
+        - ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
+      meta: end_host
+
+    - name: "NB preflight | Pause 1s if OK"
+      when:
+        - nb_script.stat.exists
+        - not ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
+      delegate_to: localhost
+      pause:
+        seconds: 1
+
+  tasks:
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail telemetry)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout|default('')))
+        and ('File exists' not in (add_ip.stderr|default('')))
+
+    - name: Debug temp IP result
+      debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc|default('') }}"
+          - "add_ip.out={{ add_ip.stdout|default('')|trim }}"
+          - "add_ip.err={{ add_ip.stderr|default('')|trim }}"
+
+    - name: Discover DEV2 MAC via bridge fdb
+      raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered MAC
+      set_fact:
+        dev2_mac: "{{ dev2_mac_scan.stdout|default('')|trim }}"
+
+    - name: Clear existing ARP entry
+      raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP if MAC found
+      when: dev2_mac|length > 0
+      raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout|default('')))
+        and ('File exists' not in (dev2_arp_add.stderr|default('')))
+
+    - name: Debug ARP summary
+      debug:
+        msg:
+          - "dev2_mac={{ dev2_mac|default('unset') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc|default('') }}"
+          - "arp_add.out={{ dev2_arp_add.stdout|default('')|trim }}"
+          - "arp_add.err={{ dev2_arp_add.stderr|default('')|trim }}"
+
+    - name: Note skipping static ARP
+      when: dev2_mac|length == 0
+      debug:
+        msg: "No MAC discovered; skipping static ARP add."
+
+    - name: Refresh ARP
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Tunnel setup -----------------------
+    - name: Pick free local port
+      delegate_to: localhost
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set tunnel facts
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ pick_port.stdout|default('')|trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout|default('')|trim }}"
+        _ctrl_sock: "{{ mktemp_dir.stdout|default('')|trim }}/ssh_tunnel_ctl"
+
+    - name: Start SSH tunnel
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host|default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- Dev1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout|trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout|trim }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname|default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- Dev1 SSID count (strict then fallback) -----------------------
+    - name: Dev1 | Count SSID lines (strict 'ssid' + 'ikeja')
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+        "cat /tmp/config.json 2>/dev/null | grep ssid | grep ikeja | wc -l"
+      register: ssid_strict_count_raw
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Decide if strict count usable
+      delegate_to: localhost
+      set_fact:
+        ssid_strict_count: "{{ (ssid_strict_count_raw.stdout | default('0')) | trim | int }}"
+        ssid_strict_ok: "{{ ((ssid_strict_count_raw.stdout | default('0')) | trim | int) > 0 and (((ssid_strict_count_raw.stdout | default('0')) | trim | int) % 2 == 0) }}"
+
+    - name: Dev1 | Count SSID lines (fallback exclude bssid/auto_ssid/SC/backhaul)
+      when: not ssid_strict_ok | default(false)
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+        "cat /tmp/config.json 2>/dev/null | grep ssid | grep -vE 'bssid|auto_ssid|SC|backhaul' | wc -l"
+      register: ssid_fallback_count_raw
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Compute final SSID count
+      delegate_to: localhost
+      when:
+        - (ssid_strict_count is defined) or (ssid_fallback_count is defined)
+      set_fact:
+        ssid_final_count: >-
+          {{
+            (strict_ok | default(false))
+            | ternary(
+                (ssid_strict_count.stdout | default('0') | int) // 2,
+                (ssid_fallback_count.stdout | default('0') | int) // 2
+              )
+          }}
+
+    - name: Dev1 | SSID counting debug
+      delegate_to: localhost
+      debug:
+        msg:
+          - "strict_ok={{ strict_ok | default(false) }}"
+          - "strict_count={{ ssid_strict_count.stdout | default('n/a') }}"
+          - "fallback_count={{ ssid_fallback_count.stdout | default('n/a') }}"
+          - "ssid_final_count={{ ssid_final_count | default('n/a') }}"
+
+    - name: Dev1 | Build multissidfix value
+      delegate_to: localhost
+      when: ssid_final_count is defined
+      set_fact:
+        multissidfix_value: "{{ (ssid_final_count | int) ~ ' ssids' }}"
+
+    - name: Dev1 | multissidfix debug
+      delegate_to: localhost
+      when: multissidfix_value is defined
+      debug:
+        msg: "multissidfix_value={{ multissidfix_value }}"
+
+
+    - name: Publish Dev1 multissidfix custom field
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname|default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'multissidfix',
+            'task_result': multissidfix_value
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- Dev1 wifidebug.sh check & publish -----------------------
+    - name: Dev1 | Check wifidebug.sh presence
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+        "test -f /root/wifidebug.sh && echo present || echo missing"
+      register: wd_present
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Set presence fact
+      delegate_to: localhost
+      set_fact:
+        wifidebug_present: "{{ (wd_present.stdout | default('') | trim) == 'present' }}"
+
+    - name: Dev1 | Read wifidebug.sh version (SCRIPTVERSION)
+      when: wifidebug_present | default(false)
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+        "grep -m1 'SCRIPTVERSION' /root/wifidebug.sh 2>/dev/null | awk -F'=' '{print \$2}' | tr -d '[:space:]' | sed -E 's/^v?([0-9].*)$/\1/'"
+      register: wd_version_raw
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize wifidebug version
+      delegate_to: localhost
+      set_fact:
+        wd_version_clean: "{{ wd_version_raw.stdout | default('') | trim }}"
+
+    - name: Dev1 | Compute wifidebug value
+      delegate_to: localhost
+      set_fact:
+        wifidebug_value: >-
+          {%- if not (wifidebug_present | default(false)) -%}missing
+          {%- elif (wd_version_clean | default('') | length) > 0 -%}v{{ wd_version_clean }}
+          {%- else -%}n/a{%- endif -%}
+
+    - name: Publish Dev1 wifidebug custom field
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname|default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'wifidebug',
+            'task_result': wifidebug_value
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- Dev2 auth + firmux -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok|default(false)
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set dev2_passfile_used
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0)|ternary(dev2_auth.stdout|trim,'NONE') }}"
+
+    # ------ LLDP candidate IP when Dev2 login failed (tunnel up but Dev2 unreachable) ------
+    - name: Dev1 | Try LLDP-derived Dev2 IP (auth to Dev2 failed)
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+      delegate_to: localhost
+      shell: |
+        DIGITS="{{ inventory_hostname | regex_replace('[^0-9]', '') }}"
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+        "cat /var/run/lldp_server.json 2>/dev/null | grep ${DIGITS} -A 10 | grep address | grep -vE 'subtype|ipv6' | awk -F'\"' '{ print \$4 }' | head -n1"
+      register: lldp_ip_raw
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Show LLDP candidate IP (Dev2, after SSH auth failure)
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+      delegate_to: localhost
+      debug:
+        msg:
+          - "LLDP-derived Dev2 IP (best-effort): {{ (lldp_ip_raw.stdout | default('') | trim) if ((lldp_ip_raw.stdout | default('') | trim | length) > 0) else '<none>' }}"
+
+    # ------ NEW: Try direct SSH to LLDP IP from localhost (no extra tunnel) ------
+    - name: Set LLDP IP fact
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+      delegate_to: localhost
+      set_fact:
+        lldp_dev2_ip: "{{ lldp_ip_raw.stdout | default('') | trim }}"
+
+    - name: Try DEV2 login via direct LLDP IP
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+        - lldp_dev2_ip | length > 0
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          if timeout 20s sshpass -f "$f" ssh {{ ssh_opts_common }} \
+               -o ConnectionAttempts=1 -o ConnectTimeout=10 \
+               "{{ dev2_ssh_user }}@{{ lldp_dev2_ip }}" "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth_lldp
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    # Debug 1: what happened during LLDP auth try
+    - name: Dev2 | LLDP path selection debug
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+        - lldp_dev2_ip | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "LLDP candidate IP={{ lldp_dev2_ip }}"
+          - "LLDP auth rc={{ dev2_auth_lldp.rc | default('n/a') }}"
+          - "passfile (if any) will follow in next step"
+
+    - name: Set dev2_passfile_used_lldp
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+        - lldp_dev2_ip | length > 0
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used_lldp: "{{ (dev2_auth_lldp.rc | default(1)) == 0 | ternary(dev2_auth_lldp.stdout | default('') | trim, 'NONE') }}"
+
+    # Debug 2: which passfile will be used for firmux read
+    - name: Dev2 | LLDP passfile decision
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+        - lldp_dev2_ip | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "Using passfile for LLDP path: {{ dev2_passfile_used_lldp }}"
+
+    # Optional quick TCP check so you see port reachability (no fail)
+    - name: Dev2 | Check TCP/22 reachability (LLDP)
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+        - lldp_dev2_ip | length > 0
+      delegate_to: localhost
+      shell: "nc -z -w3 {{ lldp_dev2_ip }} 22"
+      register: lldp_ssh_tcp
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Read firmux (direct LLDP IP)
+      when:
+        - tunnel_ok | default(false)
+        - dev2_passfile_used == 'NONE'
+        - lldp_dev2_ip | length > 0
+        - dev2_passfile_used_lldp != 'NONE'
+      delegate_to: localhost
+      shell: |
+        sshpass -f "{{ dev2_passfile_used_lldp }}" ssh \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o ConnectTimeout=15 \
+          "{{ dev2_ssh_user }}@{{ lldp_dev2_ip }}" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      register: dev2_firmux_lldp
+      failed_when: false
+      ignore_errors: true
+
+    # ------ Mux: choose whichever firmux output succeeded ------
+    - name: Dev2 | Choose firmux output (tunnel or LLDP)
+      delegate_to: localhost
+      set_fact:
+        dev2_firmux_text: >-
+          {{
+            (dev2_firmux.stdout | default('') | trim)
+            if ((dev2_firmux.stdout | default('') | trim | length) > 0)
+            else (dev2_firmux_lldp.stdout | default('') | trim)
+          }}
+
+    - name: Dev2 | Extract firmware version line
+      delegate_to: localhost
+      run_once: true
+      set_fact:
+        indoor_fw_norm: >-
+          {{
+            (
+              (dev2_firmux_text | default('') | regex_replace('\r',''))|split('\n')
+              | map('trim') | select('truthy') | list
+            ) | select('match','(?i).*\\brev\\s*[0-9]+.*')
+              | list | first | default(
+                  ((dev2_firmux_text | default('') | regex_replace('\r',''))
+                   | split('\n')|map('trim')|select('truthy')|list|first|default(''))
+                ) | trim
+          }}
+
+    - name: Publish Dev2 indoor_fwver
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname|default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP (cleanup)
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster
+      delegate_to: localhost
+      shell: "ssh -S '{{ _ctrl_sock|default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove control dir
+      delegate_to: localhost
+      file:
+        path: "{{ _ctrl_dir|default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE (telemetry only)"

files/ansible-playbooks/update-indoor.yml

@@ -1,5 +1,4 @@
----
-# update-secondline.yml
+# update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
   gather_facts: no
@@ -27,7 +26,7 @@
       - "basicpass2"
 
     # SSH options used from controller
-    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
 
     # Image to stage on DEV2 (we validate first; the actual write happens later)
     image_filename: "fox200-2.2.1-r6801.bin"
@@ -50,7 +49,6 @@
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
     afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
 
-    
   pre_tasks:
     # ------------------------------- Hostname sanity DEV1 -------------------------------
     - name: Read DEV1 hostname (busybox-safe)
@@ -71,6 +69,28 @@
           - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
           - "inventory_hostname={{ inventory_hostname }}"
 
+    # ====== NEW: pick up force-upgrade flag from CLI and TEMP override ======
+    - name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes)
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | trim }}"
+
+    # >>> TEMPORARY OVERRIDE (REMOVE THIS TASK LATER) <<<
+    - name: TEMPORARILY set force-upgrade to yes (REMOVE BEFORE COMMITTING)
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "no"
+    # <<< END TEMPORARY >>>
+
+    - name: Normalize force-upgrade flag to boolean
+      ansible.builtin.set_fact:
+        force_upgrade: "{{ (force_upgrade_raw | string | trim) | bool }}"
+
+    - name: Debug force-upgrade effective
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "force_upgrade_raw={{ force_upgrade_raw | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+
     - name: Stop early if connected DEV1 hostname != inventory
       ansible.builtin.meta: end_host
       when: (dev1_host_read.stdout | trim | length > 0) and
@@ -165,10 +185,12 @@
       register: pick_port
       changed_when: false
 
+    # (moved up) Stop immediately if no free local port was found
     - name: Stop if no free local port was found
       ansible.builtin.meta: end_host
       when: (pick_port.stdout | trim | length) == 0
 
+    # (moved up) Set chosen port and control socket path
     - name: Record chosen local port and create control dir for SSH ControlMaster
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -180,15 +202,34 @@
       ansible.builtin.set_fact:
         _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
 
-    - name: Debug  tunnel parameters (controller side)
+    # (moved down) Now it’s safe to reference _local_port/_ctrl_sock
+    - name: Debug picked local port (controller)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
-          - "local_port={{ _local_port }}"
-          - "ctrl_dir={{ _ctrl_dir }}"
+          - "picked_local_port={{ _local_port }}"
           - "ctrl_sock={{ _ctrl_sock }}"
+          - "dev1_host={{ ansible_host | default(inventory_hostname) }}"
+          - "dev2_target={{ dev2_host }}:{{ dev2_port }}"
 
-    - name: Refresh ARP on DEV1’s LAN (send unsolicited ARP from temporary IP)
+    - name: Show current listeners on picked port (ss/lsof)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p'
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+      register: port_listeners_before
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (before starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "listeners_before:\n{{ (port_listeners_before.stdout | default('')) | trim }}"
+
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
       ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
 
     # ---------------------------- Start SSH local forward via DEV1 ----------------------------
@@ -207,6 +248,35 @@
       register: start_tunnel
       changed_when: true
 
+    - name: Debug ControlMaster start result (rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "start_tunnel.rc={{ start_tunnel.rc | default('NA') }}"
+          - "start_tunnel.stdout={{ (start_tunnel.stdout | default('')) | trim }}"
+          - "start_tunnel.stderr={{ (start_tunnel.stderr | default('')) | trim }}"
+
+    - name: Show who is listening now on the local port (post-start)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: port_listeners_after
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (after starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ (port_listeners_after.stdout | default('')) | trim }}"
+
     - name: Wait a moment for tunnel to settle
       delegate_to: localhost
       ansible.builtin.wait_for:
@@ -229,6 +299,14 @@
           - "tunnel_check.rc={{ tun_check.rc }}"
           - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
 
+    - name: Debug  ControlMaster check (full rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tun_check.rc={{ tun_check.rc | default('NA') }}"
+          - "tun_check.stdout={{ (tun_check.stdout | default('')) | trim }}"
+          - "tun_check.stderr={{ (tun_check.stderr | default('')) | trim }}"
+
     # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
     - name: Probe TCP reachability to DEV2 through the tunnel (nc)
       delegate_to: localhost
@@ -266,6 +344,33 @@
           - "{{ (dev2_ls.stdout | default('')) | trim }}"
           - "{{ (dev2_ls.stderr | default('')) | trim }}"
 
+    - name: Refresh ARP 2 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
+    # ---------------------------- Single banner probe (kept) ----------------------------
+    - name: Probe SSH banner through tunnel (pre-auth, quick)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        ssh -p "$PORT" \
+          -o PreferredAuthentications=none \
+          -o PubkeyAuthentication=no \
+          -o KbdInteractiveAuthentication=no \
+          -o PasswordAuthentication=no \
+          -o NumberOfPasswordPrompts=0 \
+          -o ConnectTimeout=5 \
+          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          -vvv root@127.0.0.1 true 2>&1 || true
+      register: tunnel_banner_probe
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  SSH preauth probe (first 40 lines)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "{{ (tunnel_banner_probe.stdout | default('') | split('\n'))[:40] | join('\n') }}"
+
     # ---------------------------- Pick DEV2 password for root ----------------------------
     - name: Try DEV2 login with 'basicpass' (root)
       delegate_to: localhost
@@ -276,12 +381,36 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass rc={{ dev2_try_basicpass.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass.stdout | default('')) | trim }}"
+
     - name: Select 'basicpass' if previous login succeeded
       when: dev2_try_basicpass.rc == 0
       delegate_to: localhost
@@ -299,12 +428,37 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass2
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass2 try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass2
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass2)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass2 rc={{ dev2_try_basicpass2.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass2.stdout | default('')) | trim }}"
+      when: dev2_try_basicpass2 is defined
+
     - name: Select 'basicpass2' if second login succeeded
       when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
       delegate_to: localhost
@@ -337,7 +491,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
       args:
@@ -456,6 +610,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ---------------------------- DEV2 version firmux primary check ----------------------------
     - name: Read DEV2 /usr/lib/release/firmux (if present)
       when: dev2_passfile_used != "NONE"
@@ -467,7 +624,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /usr/lib/release/firmux 2>/dev/null || true"
       args:
@@ -482,6 +639,126 @@
       ansible.builtin.debug:
         msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
 
+    # ====== INSERTED: two-step normalization preview (rev->r, then space->dash) ======
+    - name: "Normalize DEV2 firmux (step 1): replace 'rev ' -> 'r' (debug only)"
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _firmux_step1: "{{ (dev2_firmux.stdout | default('') | trim) | regex_replace('(?i)rev\\s+','r') }}"
+      changed_when: false
+
+    - name: "Debug firmux after step 1 (rev->r)"
+      when: _firmux_step1 is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "_firmux_step1={{ _firmux_step1 }}"
+
+    - name: "Normalize DEV2 firmux (step 2): replace space -> '-' (debug only)"
+      when: _firmux_step1 is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _firmux_step2: "{{ _firmux_step1 | replace(' ', '-') }}"
+      changed_when: false
+
+    - name: "Debug firmux after step 2 (space->dash)"
+      when: _firmux_step2 is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "_firmux_step2={{ _firmux_step2 }}"
+    # ====== END INSERTED ======
+
+
+    # ===================== Early equality/substring check (same logic as checker) =====================
+    # 1) expected_norm from image_filename (prefer extracted X.Y.Z-rNNNN)
+    - name: Normalize expected target step one compute base string (from image_filename)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_step1: "{{ (image_filename | default('') | trim) }}"
+      changed_when: false
+
+    - name: Extract version core X dot Y dot Z dash rNNNN from image_filename if present
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core_list: "{{ (image_filename | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
+      changed_when: false
+
+    - name: Choose first extracted core if available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
+      changed_when: false
+
+    - name: Normalize expected target step two prefer extracted core when available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+      changed_when: false
+
+    # 2) banner_raw + banner_norm (convert "rev NNNN" → "-rNNNN" only if needed)
+    - name: Capture firmware banner line from DEV2 (raw)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_raw: "{{ (dev2_firmux.stdout | default('') | trim) }}"
+      changed_when: false
+
+    - name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_norm: >-
+          {{
+            ((banner_raw | lower) is search('-r[0-9]+$'))
+            | ternary(
+                banner_raw,
+                (banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1'))
+              )
+          }}
+      changed_when: false
+
+    # 3) evaluate (same equality OR substring, case-insensitive)
+    - name: Evaluate version match using normalized equality or substring
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        version_match: >-
+          {{
+            (expected_norm | default('') | length > 0)
+            and (
+              (banner_norm | default('')) == (expected_norm | default(''))
+              or ((banner_norm | default('') | lower) is search((expected_norm | default('') | lower)))
+              or ((expected_norm | default('') | lower) is search((banner_norm | default('') | lower)))
+            )
+          }}
+      changed_when: false
+
+    - name: Debug  version compare snapshot (pre-write)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm={{ expected_norm | default('') }}"
+          - "banner_raw={{ banner_raw | default('') }}"
+          - "banner_norm={{ banner_norm | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+          - "version_match={{ version_match | default(false) }}"
+
+    # 4) soft-block when equal (unless forced) + back-fill legacy debug names
+    - name: Soft-block if already on target (skip staging/write unless force-upgrade)
+      when:
+        - not (force_upgrade | default(false) | bool)
+        - version_match | default(false)
+      ansible.builtin.set_fact:
+        _blocked: true
+        _journal: "{{ (_journal | default([])) + [ 'Device already running target image: ' ~ (banner_norm | default('')) ~ ' — skipping write/flip (use force-upgrade=yes to override)' ] }}"
+        expected_fw_core_early: "{{ expected_norm | default('') }}"
+        current_fw_core: "{{ banner_norm | default('') }}"
+      changed_when: false
+
+    # If we are NOT blocked, still fill the legacy names so your later summary prints them
+    - name: Back-fill compare names for summary (no-op if already set)
+      when: not (_blocked | default(false))
+      ansible.builtin.set_fact:
+        expected_fw_core_early: "{{ expected_fw_core_early | default(expected_norm | default('')) }}"
+        current_fw_core: "{{ current_fw_core | default(banner_norm | default('')) }}"
+      changed_when: false
+
     # ---------------------------- Normalize rebootin early (HOURS) ----------------------------
     # Strict: rebootin is integer HOURS only. Always add +20s grace to the schedule.
 
@@ -628,6 +905,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
 
+    - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # fw_printenv health before upload (soft-fail)
     - name: Read fw_printenv size (line count) on DEV2 (soft health)
       when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
@@ -663,7 +943,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
       args:
@@ -695,7 +975,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
       args:
@@ -746,7 +1026,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "update -c '{{ dev2_image_path }}' 2>&1 || true"
       args:
@@ -804,13 +1084,16 @@
       when: journal_indoor_aborted is defined
       delegate_to: localhost
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
     - name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
       when: not (_blocked | default(false))
       block:
 
         # --- supervised async write with stall handling ---
-        - name: Write image to inactive bank on DEV2 (update -w, supervised async)
+        - name: Launch update -w on DEV2 (async)
           delegate_to: localhost
           vars:
             _write_async_cap: 600     # 10 minutes max runtime (tune if needed)
@@ -982,8 +1265,6 @@
           changed_when: true
           ignore_errors: true
 
-    
-
     # ---------------------------- Reboot scheduling (normalized) ----------------------------
     - name: Schedule DEV2 reboot after computed delay (seconds)
       when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
@@ -1187,13 +1468,16 @@
       when: afterupgrade_payload is defined
       delegate_to: localhost
 
-    # ---------------------------- Final operator summary (concise) ----------------------------
+    # ---------------------------- Final operator summary (one-liners) ----------------------------
     - name: Summary key outcomes (one-liners)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
           - "dev2_passfile_used={{ dev2_passfile_used }}"
           - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+          - "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
+          - "current_fw_core={{ current_fw_core | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
           - "local_image_present={{ local_img.stat.exists | default(false) }}"
           - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
           - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"

files/ansible-playbooks/update-reboot.yml

@@ -1,4 +1,4 @@
 ---
 # update-reboot.yml — thin wrapper, no var forwarding. 
 # Delegates entirely to the unified updater.
-- import_playbook: update-rebootin222.yml
+- import_playbook: update-rebootin223.yml

files/ansible-playbooks/update-rebootin223.yml

@@ -0,0 +1,809 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.3-r9800.bin
+    firmware_sha256: "5c06496a896831c5548988ac2575a0a84bb9054a769159dec354e1f996c252e4"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.3-r9800.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: "{{ au_delay_sec | default(300) }}"
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-upgrade payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterupgrade_payload={{ afterupgrade_payload | to_nice_json }}"
+              - "au_delay_sec={{ au_delay_sec }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost

files/nb_onedevice_update.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""
+nb_sync_one_device.py
+
+Update a single NetBox device from Cloud by hostname — ONLY if device is online.
+
+Behavior
+- Liveness gate via /v1/devices/{cloud_id}/liveness (no changes if offline).
+- Updates NetBox fields from Cloud:
+  custom_fields.fw_version ← firmwareVersion
+  custom_fields.nodeName   ← nodeName
+  custom_fields.sectorName ← sectorName
+  custom_fields.smallCellName ← smallCellName
+  serial ← serialNumber
+- IP handling:
+  If Cloud ipAddress is valid (not None/""/"0.0.0.0"):
+    ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
+    then PRUNE all other IPs on this device (default).
+  If Cloud ipAddress is placeholder/invalid:
+    skip IP changes and do not prune.
+- --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
+- NEW: Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+
+Exit codes:
+  0 = success
+  1 = not found / missing data / offline
+  3 = network/HTTP error
+  4 = NetBox update error
+"""
+
+import sys
+import json
+import argparse
+from typing import Optional, Union, List, Dict
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# ------------ HARD-CODED CONFIG (per Pavel) ------------
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v1/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzYxNzkwNDgwLCJleHAiOjE3NjQzODI0ODB9.DslgtHCCxYaykWG_1DcR2dg4mjKFCn50Ucpg3xpGAN0"
+
+REQ_TIMEOUT = 30
+CHATTY = False
+
+# ------------ HTTP utilities ------------
+def _new_session() -> requests.Session:
+    s = requests.Session()
+    retries = Retry(
+        total=3, connect=3, read=3, status=3,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+        respect_retry_after_header=True,
+    )
+    adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
+    s.mount('http://', adapter)
+    s.mount('https://', adapter)
+    s.headers['Accept'] = 'application/json'
+    return s
+
+S_NB = _new_session()
+S_CL = _new_session()
+
+# ------------ Logging / status helpers ------------
+def log(msg: str):
+    if CHATTY:
+        print(msg, file=sys.stderr)
+
+def die(code: int, msg: str):
+    # Single-line result: stdout on success, stderr on failure
+    print(msg, file=sys.stdout if code == 0 else sys.stderr)
+    raise SystemExit(code)
+
+# ------------ NetBox API ------------
+class NetBox:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip('/')
+        self.token = token
+
+    def _h(self):
+        return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def get_device_by_name(self, name: str) -> Optional[dict]:
+        log(f"NB: lookup device by name {name}")
+        r = S_NB.get(self._url("/api/dcim/devices/"), headers=self._h(),
+                     params={"name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET devices name={name} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def get_device(self, dev_id: int) -> dict:
+        log(f"NB: fetch device id={dev_id}")
+        r = S_NB.get(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET device id={dev_id} HTTP={r.status_code}")
+        return r.json()
+
+    def patch_device(self, dev_id: int, patch: dict) -> None:
+        if not patch:
+            log("NB: no device patch needed")
+            return
+        log(f"NB: patch device id={dev_id} keys={list(patch.keys())}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps(patch), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL patch device dev={dev_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                     params={"device_id": dev_id, "name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET interfaces device_id={dev_id} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0]["id"] if res else None
+
+    def ensure_eth0(self, dev_id: int) -> int:
+        ifid = self.get_iface_id(dev_id, "eth0")
+        log(f"NB: ensure eth0 (current id={ifid})")
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = S_NB.post(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        if r.status_code == 400:
+            # race: read again
+            ifid = self.get_iface_id(dev_id, "eth0")
+            if ifid:
+                return ifid
+        die(4, f"FAIL create eth0 HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_ip_by_addr(self, addr: str) -> Optional[dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"address": f"{addr}/32"}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET ip-addresses addr={addr} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def create_ip_for_iface(self, addr: str, iface_id: int) -> int:
+        payload = {
+            "address": f"{addr}/32",
+            "status": "active",
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        }
+        r = S_NB.post(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        die(4, f"FAIL create IP {addr} HTTP={r.status_code} body={r.text[:200]}")
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
+        r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
+                       data=json.dumps({
+                           "assigned_object_type": "dcim.interface",
+                           "assigned_object_id": iface_id
+                       }),
+                       timeout=REQ_TIMEOUT)
+        return 200 <= r.status_code < 300
+
+    def device_set_primary_ip4(self, dev_id: int, ip_id: int) -> None:
+        log(f"NB: set primary_ip4 dev={dev_id} -> ip_id={ip_id}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps({"primary_ip4": ip_id}), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL set primary_ip4 dev={dev_id} ip_id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = S_NB.get(self._url(f"/api/dcim/interfaces/{iface_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return (r.json().get("device") or {}).get("id")
+        return None
+
+    # --- Helpers for pruning ---
+    def list_device_ips(self, dev_id: int) -> List[Dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return r.json().get("results") or []
+        ips: List[Dict] = []
+        r2 = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r2.status_code == 200:
+            for iface in (r2.json().get("results") or []):
+                ifid = iface.get("id")
+                r3 = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                              params={"assigned_object_type": "dcim.interface",
+                                      "assigned_object_id": ifid, "limit": 1000}, timeout=REQ_TIMEOUT)
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get("results") or [])
+        return ips
+
+    def delete_ip(self, ip_id: int) -> None:
+        log(f"NB: delete IP id={ip_id}")
+        r = S_NB.delete(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300 or r.status_code == 204):
+            die(4, f"FAIL delete IP id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+# ------------ Cloud API ------------
+class Cloud:
+    def __init__(self, base: str, bearer: str):
+        self.base = base.rstrip("/")
+        self.bearer = bearer
+
+    def _h(self):
+        return {"Authorization": f"Bearer {self.bearer}", "Accept": "application/json"}
+
+    def device_detail(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}"
+        log(f"CL: fetch detail cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id} HTTP={r.status_code}")
+        return r.json()
+
+    def device_liveness(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}/liveness"
+        log(f"CL: fetch liveness cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
+        return r.json()
+
+# ------------ Core ------------
+def _primary_ip4_text(dev_json: dict) -> Optional[str]:
+    p = dev_json.get("primary_ip4") or {}
+    addr = p.get("address")
+    if isinstance(addr, str) and addr.endswith("/32"):
+        return addr[:-3]
+    return addr
+
+def run(hostname: str) -> None:
+    nb = NetBox(NB_URL, NB_TOKEN)
+    cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+
+    dev = nb.get_device_by_name(hostname)
+    if not dev:
+        die(1, f"FAIL {hostname} not found in NetBox")
+
+    dev_id = dev["id"]
+    dev_full = nb.get_device(dev_id)
+    cf = dev_full.get("custom_fields") or {}
+
+    # NEW: log upgrade command if present
+    upgrade_cmd = cf.get("upgrade_cmd")
+    log(f"NB: upgrade_cmd={upgrade_cmd}")
+
+    cloud_id = cf.get("cloud_id")
+    if cloud_id in (None, "", "null"):
+        die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
+
+    # 1) Liveness gate
+    live = cl.device_liveness(cloud_id)
+    if not bool(live.get("isConnected")):
+        die(1, f"FAIL {hostname} cloud_id={cloud_id} device is not online")
+
+    current_nb_ip = _primary_ip4_text(dev_full)
+
+    # 2) Cloud detail
+    d = cl.device_detail(cloud_id)
+    fw = d.get("firmwareVersion") or d.get("version")
+    ip_from_cloud = (d.get("ipAddress") or "").strip() if isinstance(d.get("ipAddress"), str) else d.get("ipAddress")
+    node = d.get("nodeName")
+    sector = d.get("sectorName")
+    small = d.get("smallCellName")
+    serial = d.get("serialNumber")
+
+    if not fw:
+        die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
+
+    # 3) NetBox patch (idempotent)
+    cf_patch = {}
+    if cf.get("fw_version") != fw:
+        cf_patch["fw_version"] = fw
+        log(f"CF: fw_version -> {fw}")
+    if node and cf.get("nodeName") != node:
+        cf_patch["nodeName"] = node
+        log(f"CF: nodeName -> {node}")
+    if sector and cf.get("sectorName") != sector:
+        cf_patch["sectorName"] = sector
+        log(f"CF: sectorName -> {sector}")
+    if small and cf.get("smallCellName") != small:
+        cf_patch["smallCellName"] = small
+        log(f"CF: smallCellName -> {small}")
+
+    dev_patch = {}
+    if serial and (dev_full.get("serial") != serial):
+        dev_patch["serial"] = serial
+        log(f"DEV: serial -> {serial}")
+    if cf_patch:
+        dev_patch["custom_fields"] = cf_patch
+    nb.patch_device(dev_id, dev_patch)
+
+    # 4) IP handling (skip if placeholder)
+    ip_is_placeholder = (ip_from_cloud in (None, "", "0.0.0.0"))
+    ip_out_for_status = current_nb_ip  # default to current NB IP
+
+    if ip_is_placeholder:
+        log(f"IP: cloud reported placeholder '{ip_from_cloud}', skipping IP changes; keeping NetBox ip(s) as-is")
+    else:
+        iface_id = nb.ensure_eth0(dev_id)
+
+        ip_rec = nb.get_ip_by_addr(ip_from_cloud)
+        if ip_rec is None:
+            log(f"IP: create new {ip_from_cloud} on iface {iface_id}")
+            ip_id = nb.create_ip_for_iface(ip_from_cloud, iface_id)
+        else:
+            ip_id = ip_rec["id"]
+            assigned_type = ip_rec.get("assigned_object_type") or ""
+            assigned_id = ip_rec.get("assigned_object_id")
+            if not assigned_type:
+                if not nb.assign_ip_to_iface(ip_id, iface_id):
+                    die(4, f"FAIL assign IP {ip_from_cloud} to iface {iface_id}")
+            elif assigned_type == "dcim.interface":
+                if str(assigned_id) != str(iface_id):
+                    other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
+                    log(f"IP: moving {ip_from_cloud} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id}")
+                    # Clear old device primary if necessary
+                    if other_dev:
+                        r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
+                        if r.status_code == 200:
+                            old_primary_id = (r.json().get("primary_ip4") or {}).get("id")
+                            if str(old_primary_id) == str(ip_id):
+                                S_NB.patch(nb._url(f"/api/dcim/devices/{other_dev}/"),
+                                           headers=nb._h(), data=json.dumps({"primary_ip4": None}),
+                                           timeout=REQ_TIMEOUT)
+                    if not nb.assign_ip_to_iface(ip_id, iface_id):
+                        die(4, f"FAIL move IP {ip_from_cloud} to iface {iface_id}")
+            else:
+                die(4, f"FAIL IP {ip_from_cloud} assigned to {assigned_type}")
+
+        nb.device_set_primary_ip4(dev_id, ip_id)
+        ip_out_for_status = ip_from_cloud
+
+        # PRUNE all other IPs on this device (default behavior)
+        all_ips = nb.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get("id")
+            if str(rid) == str(ip_id):
+                continue
+            addr = rec.get("address")
+            log(f"IP: pruning stale {addr} (id={rid}) from device {dev_id}")
+            nb.delete_ip(rid)
+
+    print(f"OK {hostname} ip={ip_out_for_status or 'NONE'} fw={fw} node={node} sector={sector} small={small}")
+
+# ------------ CLI ------------
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname (only if online)")
+    ap.add_argument("hostname", help="Device name in NetBox")
+    ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
+    args = ap.parse_args()
+
+    CHATTY = bool(args.chatty)  # module-scope assignment
+    try:
+        run(args.hostname)
+    except requests.RequestException as e:
+        die(3, f"FAIL network error: {e}")

files/rabbit-client.sh

@@ -116,8 +116,8 @@ dispatch_task() {
     # now, today 01:00, tomorrow 01:00 (local time)
     local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
     now_s="$(date +%s)"
-    today1_s="$(date -d 'today 01:00' +%s)"
-    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    today1_s="$(date -d 'today 00:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 00:00' +%s)"
     if (( now_s < today1_s )); then
       next1_s="$today1_s"
     else
@@ -127,7 +127,7 @@ dispatch_task() {
     # Ceil hours so current minutes are preserved as in your examples
     ceil_h=$(( (diff_s + 3599) / 3600 ))
     rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
-    total_h=$(( ceil_h + rnd ))
+    total_h=$(( ceil_h + rnd - 1 ))
 
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
@@ -145,6 +145,11 @@ dispatch_task() {
     base_task="update-indoor"
     extra_nbplay_opts+=("-erebootin=${n}")
     log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})"
+  elif [[ "$task" =~ ^update-indoor-bootenv_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor-bootenv"
+    extra_nbplay_opts+=("-erebootin=${n}")
+    log "Parsed update-indoor-bootenv suffix: ${n}h (passed as -e rebootin=${n})"
   fi
 
   # --- Default rebootin for reboot-family when not explicitly provided ---