From e644135600b607915592d981768fcf08537df0a2 Mon Sep 17 00:00:00 2001
From: pavel <pavel.larkin@ikeja.co.za>
Date: Mon, 2 Feb 2026 14:00:34 +0200
Subject: [PATCH] 1400

---
 files/ansible-playbooks/update-reboot.yml     |   2 +-
 .../ansible-playbooks/update-rebootin224.yml  | 809 ++++++++++++++++++
 2 files changed, 810 insertions(+), 1 deletion(-)
 create mode 100644 files/ansible-playbooks/update-rebootin224.yml

diff --git a/files/ansible-playbooks/update-reboot.yml b/files/ansible-playbooks/update-reboot.yml
index 8b8b48f..dbd556a 100644
--- a/files/ansible-playbooks/update-reboot.yml
+++ b/files/ansible-playbooks/update-reboot.yml
@@ -1,4 +1,4 @@
 ---
 # update-reboot.yml — thin wrapper, no var forwarding. 
 # Delegates entirely to the unified updater.
-- import_playbook: update-rebootin223.yml
+- import_playbook: update-rebootin224.yml
diff --git a/files/ansible-playbooks/update-rebootin224.yml b/files/ansible-playbooks/update-rebootin224.yml
new file mode 100644
index 0000000..9eac6bf
--- /dev/null
+++ b/files/ansible-playbooks/update-rebootin224.yml
@@ -0,0 +1,809 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.4-r9850.bin
+    firmware_sha256: "38f7dd3bb5b06a2267d7bc68e2d8351df59c2aea858d644909208e11a3970539"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.4-r9850.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: "{{ au_delay_sec | default(300) }}"
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-upgrade payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterupgrade_payload={{ afterupgrade_payload | to_nice_json }}"
+              - "au_delay_sec={{ au_delay_sec }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost