1131

files/ansible-playbooks/update-outdoor-bootenv.yml

@@ -0,0 +1,792 @@
+- name: Fix outdoor bootenv safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # Future post-bootenv check scheduling (kept disabled for now)
+    afterbootenv_hold_exchange: "{{ lookup('env','AFTERBOOTENV_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterbootenv_routing_key: "{{ lookup('env','AFTERBOOTENV_ROUTING_KEY') | default('deviceconfig', true) }}"
+    afterbootenv_hold_queue: "{{ lookup('env','AFTERBOOTENV_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+    afterbootenv_check_enabled: false
+
+    bootenv_filename: "fox100_bootenv.bin"
+    bootenv_sha256: "324337e20b0a2d8048c359bfa2a1b8dffd6b1a28eab260143dd895ca39c034aa"
+    bootenv_path: "/tmp/{{ bootenv_filename }}"
+    bootenv_mtd_device: "/dev/mtdblock8"
+    bootenv_expected_size: "262144"
+
+    # Helper computed vars
+    bootenv_name: "{{ bootenv_filename | regex_replace('\\.bin$', '') }}"
+    bootenv_marker: "/tmp/bootenv_fixed_{{ bootenv_name }}"
+    bootenv_lock_marker: "/tmp/bootenv_fix_inprogress_{{ bootenv_name }}"
+    firmware_guard_marker: "/tmp/prepared_for_{{ bootenv_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping bootenv steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Bootenv fix cancelled because a prepared marker is already present; " ~
+                  "expected marker " ~ firmware_guard_marker ~ ". Skipping bootenv steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    - name: Check if bootenv lock marker already exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e '{{ bootenv_lock_marker }}' ] && echo PRESENT || echo ABSENT"
+      register: bootenv_lock_scan
+      changed_when: false
+
+    - name: Debug bootenv lock marker presence
+      ansible.builtin.debug:
+        msg: "bootenv_lock_marker={{ bootenv_lock_scan.stdout | trim }}"
+
+    - name: Journal bootenv fix already in progress, skipping
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Bootenv fix cancelled because lock marker already exists; " ~
+                  "marker " ~ bootenv_lock_marker ~ ". Another process may be running."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_lock_present
+      changed_when: (rmq_journal_lock_present.json is defined) and (rmq_journal_lock_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_lock_present.status != 200) or
+        (rmq_journal_lock_present.json is not defined) or
+        (not (rmq_journal_lock_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
+
+    - name: End play for this host (bootenv lock already present)
+      ansible.builtin.meta: end_host
+      when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
+
+    - name: Create bootenv lock marker
+      ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_lock_marker }}'"
+      changed_when: true
+
+    - name: Bootenv fix main block
+      block:
+        - name: Count fw_printenv lines before bootenv write
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count_before
+          changed_when: false
+
+        - name: Debug fw_printenv line count before bootenv write
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines_before={{ env_line_count_before.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid before bootenv write (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count_before.stdout | trim }} lines (<7) before bootenv write — environment missing or corrupted."
+          when: (env_line_count_before.stdout | trim | int) < 7
+
+        - name: Check if bootenv image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ bootenv_path }}' ] && echo OK || echo MISSING"
+          register: bootenv_exists
+          changed_when: false
+
+        - name: Upload bootenv to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ bootenv_src_local | default("/opt/containers/ansible-worker/files/fox100_bootenv.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ bootenv_path }}"
+          delegate_to: localhost
+          when: bootenv_exists.stdout is not defined or (bootenv_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check bootenv presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ bootenv_path }}' && echo OK || echo MISSING"
+          register: bootenv_exists2
+          changed_when: false
+          failed_when: (bootenv_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded bootenv image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ bootenv_path }}' | awk '{print $1}'"
+          register: bootenv_sha_out
+          changed_when: false
+
+        - name: Verify bootenv sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ bootenv_path }}. Got {{ bootenv_sha_out.stdout | trim }}, expected {{ bootenv_sha256 }}"
+          when: (bootenv_sha_out.stdout | trim) != (bootenv_sha256 | trim)
+
+        - name: Bootenv sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ bootenv_sha_out.stdout | trim }}"
+
+        - name: Read bootenv image size on target
+          ansible.builtin.raw: "{{ pathprefix }} wc -c < '{{ bootenv_path }}'"
+          register: bootenv_size_out
+          changed_when: false
+
+        - name: Verify bootenv image size matches expected
+          ansible.builtin.fail:
+            msg: "Bootenv size mismatch for {{ bootenv_path }}. Got {{ bootenv_size_out.stdout | trim }}, expected {{ bootenv_expected_size }}"
+          when: (bootenv_size_out.stdout | trim) != (bootenv_expected_size | string | trim)
+
+        - name: Bootenv size verification debug
+          ansible.builtin.debug:
+            msg:
+              - "bootenv_size={{ bootenv_size_out.stdout | trim }}"
+
+        - name: Write bootenv to flash with dd
+          ansible.builtin.raw: "{{ pathprefix }} dd if='{{ bootenv_path }}' of={{ bootenv_mtd_device }}"
+          register: bootenv_dd
+          changed_when: true
+          failed_when: bootenv_dd.rc != 0
+
+        - name: Debug dd output (bootenv)
+          ansible.builtin.debug:
+            msg:
+              - "dd.rc={{ bootenv_dd.rc | default('NA') }}"
+              - "dd.stdout={{ (bootenv_dd.stdout | default('')) | trim }}"
+              - "dd.stderr={{ (bootenv_dd.stderr | default('')) | trim }}"
+
+        - name: Run sync after bootenv write
+          ansible.builtin.raw: "{{ pathprefix }} sync"
+          register: bootenv_sync
+          changed_when: true
+          failed_when: bootenv_sync.rc != 0
+
+        - name: Debug sync output (bootenv)
+          ansible.builtin.debug:
+            msg:
+              - "sync.rc={{ bootenv_sync.rc | default('NA') }}"
+              - "sync.stdout={{ (bootenv_sync.stdout | default('')) | trim }}"
+              - "sync.stderr={{ (bootenv_sync.stderr | default('')) | trim }}"
+
+        - name: Set bootenv write success flag
+          ansible.builtin.set_fact:
+            _bootenv_write_success: "{{ (bootenv_dd.rc | default(1)) == 0 and (bootenv_sync.rc | default(1)) == 0 }}"
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ firmware_guard_marker }}'"
+          changed_when: true
+
+        - name: Create bootenv marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target bootenv)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "bootenv_outdoor_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ bootenv_filename }}"
+          when: _bootenv_write_success | bool
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Bootenv target file: {{ bootenv_filename }}"
+              - "Bootenv SHA256: OK ({{ bootenv_sha_out.stdout | trim }})"
+              - "Bootenv size: {{ bootenv_size_out.stdout | trim }} bytes"
+              - "fw_printenv lines before write: {{ env_line_count_before.stdout | trim }}"
+              - "dd: OK"
+              - "sync: OK"
+              - "Marker: {{ bootenv_marker }}"
+              - "Guard marker: {{ firmware_guard_marker }}"
+
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Bootenv fix complete using {{ bootenv_filename }} written to {{ bootenv_mtd_device }}.
+                Original fw_printenv line count was {{ env_line_count_before.stdout | trim }}.
+                SHA256 OK {{ bootenv_sha_out.stdout | trim }}.
+                Size {{ bootenv_size_out.stdout | trim }} bytes.
+                Marker {{ bootenv_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate bootenv change.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate bootenv change.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Init after-bootenv scheduling vars
+          ansible.builtin.set_fact:
+            ab_attempt: 1
+            ab_max_attempts: 3
+            ab_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-bootenv check payload (attempt 1)
+          ansible.builtin.set_fact:
+            ab_attempt: 1
+            ab_delay_sec: "{{ ab_delay_sec | default(300) }}"
+            ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterbootenv_payload:
+              task_name: "afterbootenv_outdoor_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_bootenv_file: "{{ bootenv_filename }}"
+              original_fw_printenv_lines: "{{ env_line_count_before.stdout | trim }}"
+              attempt: "{{ ab_attempt | default(1) }}"
+              max_attempts: "{{ ab_max_attempts | default(3) }}"
+              current_delay_sec: "{{ ab_delay_sec | default(300) }}"
+              correlation_id: "{{ ab_correlation_id }}"
+              original_emitted_at: "{{ ab_original_emitted_at }}"
+              schema_version: 1
+              is_run_by: "{{ is_run_by_effective }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (ab_delay_sec | int) * 1000 }}"
+          when: afterbootenv_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-bootenv payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterbootenv_payload={{ afterbootenv_payload | to_nice_json }}"
+              - "ab_delay_sec={{ ab_delay_sec }}"
+              - "afterbootenv_check_enabled={{ afterbootenv_check_enabled }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+        - name: Bandaid | Pause before Publish delayed after-bootenv check to holding exchange
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when:
+            - afterbootenv_payload is defined
+            - afterbootenv_check_enabled | bool
+
+        - name: Publish delayed after-bootenv check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (ab_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterbootenv_routing_key }}"
+              payload: "{{ afterbootenv_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterbootenv_resp
+          changed_when: (rmq_afterbootenv_resp.json is defined) and (rmq_afterbootenv_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterbootenv_resp.status != 200) or
+            (rmq_afterbootenv_resp.json is not defined)
+          when:
+            - afterbootenv_payload is defined
+            - afterbootenv_check_enabled | bool
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state waiting (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'waiting'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Compute action_restart_timestamp (unix seconds) when rebootin == 0
+          ansible.builtin.set_fact:
+            action_restart_timestamp: "{{ lookup('pipe','date -u +%s') | int }}"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - reboot_seconds is defined
+            - (reboot_seconds | int) == 0
+
+        - name: Scheduler | Compute action_restart_timestamp (unix seconds) when reboot scheduled
+          ansible.builtin.set_fact:
+            action_restart_timestamp: "{{ (lookup('pipe','date -u +%s') | int) + (reboot_seconds | int) }}"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - reboot_seconds is defined
+            - (reboot_seconds | int) > 0
+
+        - name: Scheduler | Debug computed action_restart_timestamp
+          ansible.builtin.debug:
+            msg: "action_restart_timestamp={{ action_restart_timestamp }} (reboot_seconds={{ reboot_seconds | int }})"
+          when:
+            - is_run_by_effective == 'scheduler'
+            - action_restart_timestamp is defined
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_restart_timestamp custom field
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_restart_timestamp',
+                'task_result': (action_restart_timestamp | string)
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - action_restart_timestamp is defined
+
+        - name: Remove bootenv lock marker after success
+          ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
+          changed_when: true
+
+      rescue:
+        - name: Remove bootenv lock marker after failure
+          ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
+          changed_when: true
+          ignore_errors: true
+
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (bootenv failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (bootenv failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (bootenv failure)"
+          when: is_run_by_effective == 'scheduler'