--- - name: Upgrade firmware safely (no Python on target) hosts: all gather_facts: no vars: # RabbitMQ (pull from env if provided) rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" # NEW: Post-upgrade check scheduling (via holding queue -> DLX) # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'. # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms). # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key, # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side). afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}" afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" # Queue name is not used for publish; present for documentation/reference only afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}" # REQUIRED (pass via -e) firmware_path: /tmp/2.2.3-r9800.bin firmware_sha256: "5c06496a896831c5548988ac2575a0a84bb9054a769159dec354e1f996c252e4" # Helper computed vars fw_base: "{{ firmware_path | basename }}" fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}" fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}" fw_marker: "/tmp/prepared_for_{{ fw_name }}" pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " tasks: # ----------------------------- HOSTNAME PREFLIGHT ----------------------------- - name: Hostname preflight block: - name: Read remote HOSTNAME ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\"" register: host_env changed_when: false - name: Debug hostnames ansible.builtin.debug: msg: - "remote_hostname={{ host_env.stdout | trim }}" - "inventory_hostname={{ inventory_hostname }}" - name: Stop if connected hostname differs from inventory ansible.builtin.fail: msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'." when: (host_env.stdout | trim) != inventory_hostname rescue: - name: Build failure task name and detail (hostname preflight) ansible.builtin.set_fact: fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}" fail_detail_raw: >- {{ ansible_failed_result.msg | default(ansible_failed_result.stderr) | default(ansible_failed_result.stdout) | default('no additional error output') | trim }} - name: Build failure summary text (hostname preflight) ansible.builtin.set_fact: fail_summary: >- Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }} - name: Truncate failure summary to ~400 chars (hostname preflight) ansible.builtin.set_fact: fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}" - name: Build control queue payload for failure journal (hostname preflight) ansible.builtin.set_fact: journal_failure_payload_pre: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: "{{ fail_summary_short }}" - name: Publish failure journal to control queue (hostname preflight) ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_failure_payload_pre | to_json }}" payload_encoding: "string" register: rmq_journal_pre_resp changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_pre_resp.status != 200) or (rmq_journal_pre_resp.json is not defined) or (not (rmq_journal_pre_resp.json.routed | default(false) | bool)) delegate_to: localhost - name: Build control queue payload for update-aborted tag (hostname preflight) ansible.builtin.set_fact: tag_failed_payload_pre: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "update-aborted" - name: Publish update-aborted tag to control queue (hostname preflight) ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_failed_payload_pre | to_json }}" payload_encoding: "string" register: rmq_tag_failed_pre_resp changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_failed_pre_resp.status != 200) or (rmq_tag_failed_pre_resp.json is not defined) or (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)) delegate_to: localhost - name: Stop play after hostname preflight failure ansible.builtin.meta: end_play # --- Tag device as update-in-progress at start --- - name: Build control queue payload for update-in-progress tag ansible.builtin.set_fact: tag_inprogress_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "update-in-progress" - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_inprogress_payload | to_json }}" payload_encoding: "string" register: rmq_tag_inprogress_resp changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_inprogress_resp.status != 200) or (rmq_tag_inprogress_resp.json is not defined) or (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool)) delegate_to: localhost - name: Log control queue tag publish result ansible.builtin.debug: var: rmq_tag_inprogress_resp.json when: rmq_tag_inprogress_resp is defined # --------------------- Prepared marker check BEFORE SSID scan ----------------- - name: Check if any prepared marker exists ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT" register: prep_scan changed_when: false - name: Debug marker presence ansible.builtin.debug: msg: "prepared_marker={{ prep_scan.stdout | trim }}" - name: Journal preparation already present, skipping update steps ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "journal_add", "task_result": ( "Preparation already present for " ~ fw_banner_repr ~ "; marker " ~ fw_marker ~ ". Skipping update steps." ) } | to_json }} payload_encoding: "string" register: rmq_journal_prep_present changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool) failed_when: > (rmq_journal_prep_present.status != 200) or (rmq_journal_prep_present.json is not defined) or (not (rmq_journal_prep_present.json.routed | default(false) | bool)) delegate_to: localhost when: (prep_scan.stdout | trim) == 'PRESENT' - name: End play for this host (already prepared) ansible.builtin.meta: end_host when: (prep_scan.stdout | trim) == 'PRESENT' # --- SSID scan & journal (does not stop the play) --- - name: Count SSID lines in /tmp/config.json (filtered) ansible.builtin.raw: > {{ pathprefix }} grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l register: ssid_lines changed_when: false - name: Debug SSID count ansible.builtin.debug: msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}" - name: Build joined SSID list when multiple SSIDs found (≥3) ansible.builtin.raw: > {{ pathprefix }} grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \ | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \ | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }' register: ssid_concat changed_when: false when: (ssid_lines.stdout | trim | int) >= 3 - name: Build control queue payload for SSID journal (journal_add) ansible.builtin.set_fact: ssid_journal_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}" when: (ssid_lines.stdout | trim | int) >= 3 - name: Publish SSID journal to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ ssid_journal_payload | to_json }}" payload_encoding: "string" register: rmq_ssid_journal_resp changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool) failed_when: > (rmq_ssid_journal_resp.status != 200) or (rmq_ssid_journal_resp.json is not defined) or (not (rmq_ssid_journal_resp.json.routed | default(false) | bool)) delegate_to: localhost when: ssid_journal_payload is defined # ----------------------------- MAIN UPDATE BLOCK ----------------------------- - name: Firmware update main block block: - name: Check if firmware image is already on the device ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING" register: fw_exists changed_when: false - name: Count fw_printenv lines ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l" register: env_line_count changed_when: false - name: Debug fw_printenv line count ansible.builtin.debug: msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}" - name: Stop if bootloader environment looks invalid (<7 lines) ansible.builtin.fail: msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted." when: (env_line_count.stdout | trim | int) < 7 - name: Read first line of /etc/banner (current running version) ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1" register: banner changed_when: false - name: current version ansible.builtin.debug: msg: - "current banner: {{ banner.stdout | trim }}" - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }}) ansible.builtin.fail: msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})" when: banner.stdout is search(fw_banner_repr) - name: Upload firmware to /tmp via scp (controller-side) ansible.builtin.shell: | set -e SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.3-r9800.bin") }}' DST_USER='{{ ansible_user | default("root") }}' DST_HOST='{{ ansible_host | default(inventory_hostname) }}' test -f "$SRC" sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}" delegate_to: localhost when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK' changed_when: true - name: Re-check firmware presence after optional upload ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING" register: fw_exists2 changed_when: false failed_when: (fw_exists2.stdout | trim) != 'OK' - name: Compute sha256 of the uploaded image ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'" register: sha_out changed_when: false - name: Verify sha256 matches expected ansible.builtin.fail: msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}" when: (sha_out.stdout | trim) != (firmware_sha256 | trim) - name: sha256 verification debug ansible.builtin.debug: msg: - "sha256sum is: {{ sha_out.stdout | trim }}" - name: Check image validity (update -c must say 'valid') ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'" register: up_check changed_when: false failed_when: up_check.stdout.strip() != 'valid' - name: image verification debug ansible.builtin.debug: msg: - ".bin verification is: {{ up_check.stdout | trim }}" # - name: forced stop before writing # ansible.builtin.meta: end_play - name: Write image (this will take a while) ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'" register: up_write changed_when: true failed_when: up_write.stdout is not search('update is complete') - name: Read current active partition ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'" register: active_before changed_when: false failed_when: active_before.stdout | trim not in ['1','2'] - name: Determine new active value ansible.builtin.set_fact: new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}" - name: Switch active partition to {{ new_active }} ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}" register: setenv_out changed_when: true - name: Verify active partition flipped ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'" register: active_after changed_when: false failed_when: (active_after.stdout | trim) != new_active - name: Create prepared marker ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'" changed_when: true - name: Build control queue payload (progress & target version) ansible.builtin.set_fact: nbq2_payload_obj: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "update_wo_restart" task_result: "waiting_restart" task_add1: "{{ fw_banner_repr }}" # e.g., "2.2.0 rev 9739" when: up_write is changed - name: Publish message to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ nbq2_payload_obj | to_json }}" payload_encoding: "string" register: rmq_resp changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false)) failed_when: > (rmq_resp.status != 200) or (rmq_resp.json is not defined) or (rmq_resp.json.routed | default(false) | bool == false) delegate_to: localhost when: nbq2_payload_obj is defined - name: Log control queue publish result ansible.builtin.debug: var: rmq_resp.json when: rmq_resp is defined - name: Summary ansible.builtin.debug: msg: - "Banner before: {{ banner.stdout | trim }}" - "Target version: {{ fw_banner_repr }}" - "SHA256: OK ({{ sha_out.stdout | trim }})" - "update -c: {{ up_check.stdout | trim }}" - "update -w: OK" - "active: {{ active_before.stdout | trim }} -> {{ new_active }}" - "Marker: {{ fw_marker }}" # --- Optional scheduled reboot (must be last device-side command) --- - name: Compute reboot delay in seconds (if rebootin provided) ansible.builtin.set_fact: reboot_seconds: "{{ (rebootin | int) * 3600 }}" when: - nbq2_payload_obj is defined - rebootin is defined - name: Schedule delayed reboot on device (HUP-safe) ansible.builtin.raw: > {{ pathprefix }} sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &' changed_when: true when: - nbq2_payload_obj is defined - reboot_seconds is defined # --- Success tag selection (ONLY CHANGE) --- - name: Build control queue payload for update-auto-restarted (rebootin == 0) ansible.builtin.set_fact: tag_auto_restarted_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "update-auto-restarted" when: - nbq2_payload_obj is defined - rebootin is defined - (rebootin | int) == 0 - name: Build control queue payload for update-restart-scheduled (rebootin >= 1) ansible.builtin.set_fact: tag_restart_scheduled_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "update-restart-scheduled" when: - nbq2_payload_obj is defined - rebootin is defined - (rebootin | int) >= 1 - name: Build control queue payload for update-waits-restart tag (no reboot scheduled) ansible.builtin.set_fact: tag_waits_restart_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "update-waits-restart" when: - nbq2_payload_obj is defined - rebootin is not defined # --- Publish chosen tag (updated names only) --- - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_waits_restart_payload | to_json }}" payload_encoding: "string" register: rmq_tag_waits_restart_resp changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_waits_restart_resp.status != 200) or (rmq_tag_waits_restart_resp.json is not defined) or (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)) delegate_to: localhost when: tag_waits_restart_payload is defined - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_auto_restarted_payload | to_json }}" payload_encoding: "string" register: rmq_tag_auto_restarted_resp changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_auto_restarted_resp.status != 200) or (rmq_tag_auto_restarted_resp.json is not defined) or (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)) delegate_to: localhost when: tag_auto_restarted_payload is defined - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_restart_scheduled_payload | to_json }}" payload_encoding: "string" register: rmq_tag_restart_scheduled_resp changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_restart_scheduled_resp.status != 200) or (rmq_tag_restart_scheduled_resp.json is not defined) or (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)) delegate_to: localhost when: tag_restart_scheduled_payload is defined # --- Journal: preparation successful (only if fully successful) --- - name: Build control queue payload for success journal ansible.builtin.set_fact: journal_success_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- Preparation complete for {{ fw_banner_repr }}. Active {{ active_before.stdout | trim }} → {{ new_active }}; marker {{ fw_marker }}. {{ ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.') if (rebootin is defined) else 'Waiting for restart to activate new firmware.' }} when: nbq2_payload_obj is defined - name: Publish success journal to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_success_payload | to_json }}" payload_encoding: "string" register: rmq_journal_success_resp changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_success_resp.status != 200) or (rmq_journal_success_resp.json is not defined) or (not (rmq_journal_success_resp.json.routed | default(false) | bool)) delegate_to: localhost # ----------------- NEW: schedule "afterupgrade_check" message ------------- # Architectural notes: # - Only schedule if preparation succeeded (nbq2_payload_obj set) # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer # by re-enqueuing fresh messages with increased delays; the producer does NOT sleep. # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms). # After TTL, the holding queue dead-letters to exchange 'deviceconfig'. - name: Init after-upgrade scheduling vars ansible.builtin.set_fact: au_attempt: 1 au_max_attempts: 3 # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m). # Because this task runs with delegate_to: localhost, read from hostvars. au_delay_sec: >- {{ ( (hostvars[inventory_hostname].reboot_seconds | default(0) | int) + 300 ) if (hostvars[inventory_hostname].reboot_seconds is defined) else 300 }} when: nbq2_payload_obj is defined delegate_to: localhost # NEW: compute values that the payload will reference - name: Generate correlation ID and original emitted timestamp ansible.builtin.set_fact: au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" when: nbq2_payload_obj is defined delegate_to: localhost - name: Build after-upgrade check payload (attempt 1) ansible.builtin.set_fact: au_attempt: 1 au_delay_sec: "{{ au_delay_sec | default(300) }}" au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" afterupgrade_payload: task_name: "afterupgrade_check" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" target_version: "{{ fw_banner_repr }}" attempt: "{{ au_attempt | default(1) }}" max_attempts: "{{ au_max_attempts | default(3) }}" current_delay_sec: "{{ au_delay_sec | default(300) }}" correlation_id: "{{ au_correlation_id }}" original_emitted_at: "{{ au_original_emitted_at }}" schema_version: 1 when: nbq2_payload_obj is defined delegate_to: localhost - name: Debug x-delay about to be sent (ms) ansible.builtin.debug: msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" when: afterupgrade_payload is defined delegate_to: localhost - name: DEBUG after-upgrade payload and timing delegate_to: localhost ansible.builtin.debug: msg: - "afterupgrade_payload={{ afterupgrade_payload | to_nice_json }}" - "au_delay_sec={{ au_delay_sec }}" - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}" - name: Publish delayed after-upgrade check to holding exchange ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" headers: x-delay: "{{ (au_delay_sec | int) * 1000 }}" routing_key: "{{ afterupgrade_routing_key }}" payload: "{{ afterupgrade_payload | to_json }}" payload_encoding: "string" register: rmq_afterupgrade_resp changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) failed_when: > (rmq_afterupgrade_resp.status != 200) or (rmq_afterupgrade_resp.json is not defined) when: afterupgrade_payload is defined delegate_to: localhost rescue: - name: Build failure task name and detail ansible.builtin.set_fact: fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}" fail_detail_raw: >- {{ ansible_failed_result.msg | default(ansible_failed_result.stderr) | default(ansible_failed_result.stdout) | default('no additional error output') | trim }} - name: Build failure summary text ansible.builtin.set_fact: fail_summary: >- Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }} - name: Truncate failure summary to ~400 chars ansible.builtin.set_fact: fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}" - name: Build control queue payload for failure journal ansible.builtin.set_fact: journal_failure_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: "{{ fail_summary_short }}" - name: Publish failure journal to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_failure_payload | to_json }}" payload_encoding: "string" register: rmq_journal_fail_resp changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_fail_resp.status != 200) or (rmq_journal_fail_resp.json is not defined) or (not (rmq_journal_fail_resp.json.routed | default(false) | bool)) delegate_to: localhost - name: Build control queue payload for update-aborted tag ansible.builtin.set_fact: tag_failed_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "update-aborted" - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_failed_payload | to_json }}" payload_encoding: "string" register: rmq_tag_failed_resp changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_failed_resp.status != 200) or (rmq_tag_failed_resp.json is not defined) or (not (rmq_tag_failed_resp.json.routed | default(false) | bool)) delegate_to: localhost