diff --git a/files/ansible-playbooks/update-outdoor-bootenv.yml b/files/ansible-playbooks/update-outdoor-bootenv.yml new file mode 100644 index 0000000..dac8c17 --- /dev/null +++ b/files/ansible-playbooks/update-outdoor-bootenv.yml @@ -0,0 +1,792 @@ +- name: Fix outdoor bootenv safely (no Python on target) + hosts: all + gather_facts: no + + vars: + # RabbitMQ (pull from env if provided) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler) + is_run_by_effective: "{{ is_run_by | default('manual') }}" + + # Future post-bootenv check scheduling (kept disabled for now) + afterbootenv_hold_exchange: "{{ lookup('env','AFTERBOOTENV_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}" + afterbootenv_routing_key: "{{ lookup('env','AFTERBOOTENV_ROUTING_KEY') | default('deviceconfig', true) }}" + afterbootenv_hold_queue: "{{ lookup('env','AFTERBOOTENV_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}" + afterbootenv_check_enabled: false + + bootenv_filename: "fox100_bootenv.bin" + bootenv_sha256: "324337e20b0a2d8048c359bfa2a1b8dffd6b1a28eab260143dd895ca39c034aa" + bootenv_path: "/tmp/{{ bootenv_filename }}" + bootenv_mtd_device: "/dev/mtdblock8" + bootenv_expected_size: "262144" + + # Helper computed vars + bootenv_name: "{{ bootenv_filename | regex_replace('\\.bin$', '') }}" + bootenv_marker: "/tmp/bootenv_fixed_{{ bootenv_name }}" + bootenv_lock_marker: "/tmp/bootenv_fix_inprogress_{{ bootenv_name }}" + firmware_guard_marker: "/tmp/prepared_for_{{ bootenv_name }}" + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + tasks: + + - name: Hostname preflight + block: + - name: Read remote HOSTNAME + ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\"" + register: host_env + changed_when: false + + - name: Debug hostnames + ansible.builtin.debug: + msg: + - "remote_hostname={{ host_env.stdout | trim }}" + - "inventory_hostname={{ inventory_hostname }}" + + - name: Stop if connected hostname differs from inventory + ansible.builtin.fail: + msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'." + when: (host_env.stdout | trim) != inventory_hostname + + rescue: + - name: Build failure task name and detail (hostname preflight) + ansible.builtin.set_fact: + fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}" + fail_detail_raw: >- + {{ ansible_failed_result.msg + | default(ansible_failed_result.stderr) + | default(ansible_failed_result.stdout) + | default('no additional error output') + | trim }} + + - name: Build failure summary text (hostname preflight) + ansible.builtin.set_fact: + fail_summary: >- + Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }} + + - name: Truncate failure summary to ~400 chars (hostname preflight) + ansible.builtin.set_fact: + fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}" + + - name: Build control queue payload for failure journal (hostname preflight) + ansible.builtin.set_fact: + journal_failure_payload_pre: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "{{ fail_summary_short }}" + + - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight) + ansible.builtin.pause: + seconds: 1 + delegate_to: localhost + changed_when: false + + - name: Publish failure journal to control queue (hostname preflight) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_failure_payload_pre | to_json }}" + payload_encoding: "string" + register: rmq_journal_pre_resp + changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_pre_resp.status != 200) or + (rmq_journal_pre_resp.json is not defined) or + (not (rmq_journal_pre_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Scheduler | Publish action_state failed (hostname preflight) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_state', + 'task_result': 'failed' + } | to_json }}" + payload_encoding: "string" + changed_when: false + delegate_to: localhost + when: is_run_by_effective == 'scheduler' + + - name: Scheduler | Debug action_state failed published (hostname preflight) + ansible.builtin.debug: + msg: "scheduler-run detected; published action_state=failed (hostname preflight)" + when: is_run_by_effective == 'scheduler' + + - name: Stop play after hostname preflight failure + ansible.builtin.meta: end_play + + - name: Scheduler context | Debug is_run_by + ansible.builtin.debug: + msg: "is_run_by={{ is_run_by_effective }}" + + - name: Scheduler | Publish action_state inprogress (scheduler-run) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_state', + 'task_result': 'inprogress' + } | to_json }}" + payload_encoding: "string" + changed_when: false + delegate_to: localhost + when: is_run_by_effective == 'scheduler' + + - name: Scheduler | Debug action_state inprogress published + ansible.builtin.debug: + msg: "scheduler-run detected; published action_state=inprogress" + when: is_run_by_effective == 'scheduler' + + - name: Check if any prepared marker exists + ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT" + register: prep_scan + changed_when: false + + - name: Debug marker presence + ansible.builtin.debug: + msg: "prepared_marker={{ prep_scan.stdout | trim }}" + + - name: Journal preparation already present, skipping bootenv steps + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ( + "Bootenv fix cancelled because a prepared marker is already present; " ~ + "expected marker " ~ firmware_guard_marker ~ ". Skipping bootenv steps." + ) + } | to_json + }} + payload_encoding: "string" + register: rmq_journal_prep_present + changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_prep_present.status != 200) or + (rmq_journal_prep_present.json is not defined) or + (not (rmq_journal_prep_present.json.routed | default(false) | bool)) + delegate_to: localhost + when: (prep_scan.stdout | trim) == 'PRESENT' + + - name: End play for this host (already prepared) + ansible.builtin.meta: end_host + when: (prep_scan.stdout | trim) == 'PRESENT' + + - name: Check if bootenv lock marker already exists + ansible.builtin.raw: "{{ pathprefix }} [ -e '{{ bootenv_lock_marker }}' ] && echo PRESENT || echo ABSENT" + register: bootenv_lock_scan + changed_when: false + + - name: Debug bootenv lock marker presence + ansible.builtin.debug: + msg: "bootenv_lock_marker={{ bootenv_lock_scan.stdout | trim }}" + + - name: Journal bootenv fix already in progress, skipping + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ( + "Bootenv fix cancelled because lock marker already exists; " ~ + "marker " ~ bootenv_lock_marker ~ ". Another process may be running." + ) + } | to_json + }} + payload_encoding: "string" + register: rmq_journal_lock_present + changed_when: (rmq_journal_lock_present.json is defined) and (rmq_journal_lock_present.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_lock_present.status != 200) or + (rmq_journal_lock_present.json is not defined) or + (not (rmq_journal_lock_present.json.routed | default(false) | bool)) + delegate_to: localhost + when: (bootenv_lock_scan.stdout | trim) == 'PRESENT' + + - name: End play for this host (bootenv lock already present) + ansible.builtin.meta: end_host + when: (bootenv_lock_scan.stdout | trim) == 'PRESENT' + + - name: Create bootenv lock marker + ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_lock_marker }}'" + changed_when: true + + - name: Bootenv fix main block + block: + - name: Count fw_printenv lines before bootenv write + ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l" + register: env_line_count_before + changed_when: false + + - name: Debug fw_printenv line count before bootenv write + ansible.builtin.debug: + msg: "fw_printenv_lines_before={{ env_line_count_before.stdout | trim }}" + + - name: Stop if bootloader environment looks invalid before bootenv write (<7 lines) + ansible.builtin.fail: + msg: "Aborting: fw_printenv returned only {{ env_line_count_before.stdout | trim }} lines (<7) before bootenv write — environment missing or corrupted." + when: (env_line_count_before.stdout | trim | int) < 7 + + - name: Check if bootenv image is already on the device + ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ bootenv_path }}' ] && echo OK || echo MISSING" + register: bootenv_exists + changed_when: false + + - name: Upload bootenv to /tmp via scp (controller-side) + ansible.builtin.shell: | + set -e + SRC='{{ bootenv_src_local | default("/opt/containers/ansible-worker/files/fox100_bootenv.bin") }}' + DST_USER='{{ ansible_user | default("root") }}' + DST_HOST='{{ ansible_host | default(inventory_hostname) }}' + test -f "$SRC" + sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "$SRC" "${DST_USER}@${DST_HOST}:{{ bootenv_path }}" + delegate_to: localhost + when: bootenv_exists.stdout is not defined or (bootenv_exists.stdout | trim) != 'OK' + changed_when: true + + - name: Re-check bootenv presence after optional upload + ansible.builtin.raw: "{{ pathprefix }} test -f '{{ bootenv_path }}' && echo OK || echo MISSING" + register: bootenv_exists2 + changed_when: false + failed_when: (bootenv_exists2.stdout | trim) != 'OK' + + - name: Compute sha256 of the uploaded bootenv image + ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ bootenv_path }}' | awk '{print $1}'" + register: bootenv_sha_out + changed_when: false + + - name: Verify bootenv sha256 matches expected + ansible.builtin.fail: + msg: "SHA256 mismatch for {{ bootenv_path }}. Got {{ bootenv_sha_out.stdout | trim }}, expected {{ bootenv_sha256 }}" + when: (bootenv_sha_out.stdout | trim) != (bootenv_sha256 | trim) + + - name: Bootenv sha256 verification debug + ansible.builtin.debug: + msg: + - "sha256sum is: {{ bootenv_sha_out.stdout | trim }}" + + - name: Read bootenv image size on target + ansible.builtin.raw: "{{ pathprefix }} wc -c < '{{ bootenv_path }}'" + register: bootenv_size_out + changed_when: false + + - name: Verify bootenv image size matches expected + ansible.builtin.fail: + msg: "Bootenv size mismatch for {{ bootenv_path }}. Got {{ bootenv_size_out.stdout | trim }}, expected {{ bootenv_expected_size }}" + when: (bootenv_size_out.stdout | trim) != (bootenv_expected_size | string | trim) + + - name: Bootenv size verification debug + ansible.builtin.debug: + msg: + - "bootenv_size={{ bootenv_size_out.stdout | trim }}" + + - name: Write bootenv to flash with dd + ansible.builtin.raw: "{{ pathprefix }} dd if='{{ bootenv_path }}' of={{ bootenv_mtd_device }}" + register: bootenv_dd + changed_when: true + failed_when: bootenv_dd.rc != 0 + + - name: Debug dd output (bootenv) + ansible.builtin.debug: + msg: + - "dd.rc={{ bootenv_dd.rc | default('NA') }}" + - "dd.stdout={{ (bootenv_dd.stdout | default('')) | trim }}" + - "dd.stderr={{ (bootenv_dd.stderr | default('')) | trim }}" + + - name: Run sync after bootenv write + ansible.builtin.raw: "{{ pathprefix }} sync" + register: bootenv_sync + changed_when: true + failed_when: bootenv_sync.rc != 0 + + - name: Debug sync output (bootenv) + ansible.builtin.debug: + msg: + - "sync.rc={{ bootenv_sync.rc | default('NA') }}" + - "sync.stdout={{ (bootenv_sync.stdout | default('')) | trim }}" + - "sync.stderr={{ (bootenv_sync.stderr | default('')) | trim }}" + + - name: Set bootenv write success flag + ansible.builtin.set_fact: + _bootenv_write_success: "{{ (bootenv_dd.rc | default(1)) == 0 and (bootenv_sync.rc | default(1)) == 0 }}" + + - name: Create prepared marker + ansible.builtin.raw: "{{ pathprefix }} touch '{{ firmware_guard_marker }}'" + changed_when: true + + - name: Create bootenv marker + ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_marker }}'" + changed_when: true + + - name: Build control queue payload (progress & target bootenv) + ansible.builtin.set_fact: + nbq2_payload_obj: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "bootenv_outdoor_wo_restart" + task_result: "waiting_restart" + task_add1: "{{ bootenv_filename }}" + when: _bootenv_write_success | bool + + - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API + ansible.builtin.pause: + seconds: 1 + delegate_to: localhost + changed_when: false + + - name: Publish message to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ nbq2_payload_obj | to_json }}" + payload_encoding: "string" + register: rmq_resp + changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false)) + failed_when: > + (rmq_resp.status != 200) or + (rmq_resp.json is not defined) or + (rmq_resp.json.routed | default(false) | bool == false) + delegate_to: localhost + when: nbq2_payload_obj is defined + + - name: Log control queue publish result + ansible.builtin.debug: + var: rmq_resp.json + when: rmq_resp is defined + + - name: Summary + ansible.builtin.debug: + msg: + - "Bootenv target file: {{ bootenv_filename }}" + - "Bootenv SHA256: OK ({{ bootenv_sha_out.stdout | trim }})" + - "Bootenv size: {{ bootenv_size_out.stdout | trim }} bytes" + - "fw_printenv lines before write: {{ env_line_count_before.stdout | trim }}" + - "dd: OK" + - "sync: OK" + - "Marker: {{ bootenv_marker }}" + - "Guard marker: {{ firmware_guard_marker }}" + + - name: Compute reboot delay in seconds (if rebootin provided) + ansible.builtin.set_fact: + reboot_seconds: "{{ (rebootin | int) * 3600 }}" + when: + - nbq2_payload_obj is defined + - rebootin is defined + + - name: Schedule delayed reboot on device (HUP-safe) + ansible.builtin.raw: > + {{ pathprefix }} + sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &' + changed_when: true + when: + - nbq2_payload_obj is defined + - reboot_seconds is defined + + - name: Build control queue payload for success journal + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Bootenv fix complete using {{ bootenv_filename }} written to {{ bootenv_mtd_device }}. + Original fw_printenv line count was {{ env_line_count_before.stdout | trim }}. + SHA256 OK {{ bootenv_sha_out.stdout | trim }}. + Size {{ bootenv_size_out.stdout | trim }} bytes. + Marker {{ bootenv_marker }}. + {{ + ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate bootenv change.') + if (rebootin is defined) + else 'Waiting for restart to activate bootenv change.' + }} + when: nbq2_payload_obj is defined + + - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API + ansible.builtin.pause: + seconds: 1 + delegate_to: localhost + changed_when: false + + - name: Publish success journal to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_journal_success_resp + changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_success_resp.status != 200) or + (rmq_journal_success_resp.json is not defined) or + (not (rmq_journal_success_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Init after-bootenv scheduling vars + ansible.builtin.set_fact: + ab_attempt: 1 + ab_max_attempts: 3 + ab_delay_sec: >- + {{ + ( + (hostvars[inventory_hostname].reboot_seconds | default(0) | int) + + 300 + ) + if (hostvars[inventory_hostname].reboot_seconds is defined) + else 300 + }} + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Build after-bootenv check payload (attempt 1) + ansible.builtin.set_fact: + ab_attempt: 1 + ab_delay_sec: "{{ ab_delay_sec | default(300) }}" + ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + afterbootenv_payload: + task_name: "afterbootenv_outdoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_bootenv_file: "{{ bootenv_filename }}" + original_fw_printenv_lines: "{{ env_line_count_before.stdout | trim }}" + attempt: "{{ ab_attempt | default(1) }}" + max_attempts: "{{ ab_max_attempts | default(3) }}" + current_delay_sec: "{{ ab_delay_sec | default(300) }}" + correlation_id: "{{ ab_correlation_id }}" + original_emitted_at: "{{ ab_original_emitted_at }}" + schema_version: 1 + is_run_by: "{{ is_run_by_effective }}" + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (ab_delay_sec | int) * 1000 }}" + when: afterbootenv_payload is defined + delegate_to: localhost + + - name: DEBUG after-bootenv payload and timing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "afterbootenv_payload={{ afterbootenv_payload | to_nice_json }}" + - "ab_delay_sec={{ ab_delay_sec }}" + - "afterbootenv_check_enabled={{ afterbootenv_check_enabled }}" + - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}" + + - name: Bandaid | Pause before Publish delayed after-bootenv check to holding exchange + ansible.builtin.pause: + seconds: 1 + delegate_to: localhost + changed_when: false + when: + - afterbootenv_payload is defined + - afterbootenv_check_enabled | bool + + - name: Publish delayed after-bootenv check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (ab_delay_sec | int) * 1000 }}" + routing_key: "{{ afterbootenv_routing_key }}" + payload: "{{ afterbootenv_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterbootenv_resp + changed_when: (rmq_afterbootenv_resp.json is defined) and (rmq_afterbootenv_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterbootenv_resp.status != 200) or + (rmq_afterbootenv_resp.json is not defined) + when: + - afterbootenv_payload is defined + - afterbootenv_check_enabled | bool + delegate_to: localhost + + - name: Scheduler | Publish action_state waiting (success) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_state', + 'task_result': 'waiting' + } | to_json }}" + payload_encoding: "string" + changed_when: false + delegate_to: localhost + when: is_run_by_effective == 'scheduler' + + - name: Scheduler | Compute action_restart_timestamp (unix seconds) when rebootin == 0 + ansible.builtin.set_fact: + action_restart_timestamp: "{{ lookup('pipe','date -u +%s') | int }}" + changed_when: false + delegate_to: localhost + when: + - is_run_by_effective == 'scheduler' + - reboot_seconds is defined + - (reboot_seconds | int) == 0 + + - name: Scheduler | Compute action_restart_timestamp (unix seconds) when reboot scheduled + ansible.builtin.set_fact: + action_restart_timestamp: "{{ (lookup('pipe','date -u +%s') | int) + (reboot_seconds | int) }}" + changed_when: false + delegate_to: localhost + when: + - is_run_by_effective == 'scheduler' + - reboot_seconds is defined + - (reboot_seconds | int) > 0 + + - name: Scheduler | Debug computed action_restart_timestamp + ansible.builtin.debug: + msg: "action_restart_timestamp={{ action_restart_timestamp }} (reboot_seconds={{ reboot_seconds | int }})" + when: + - is_run_by_effective == 'scheduler' + - action_restart_timestamp is defined + delegate_to: localhost + + - name: Scheduler | Publish action_restart_timestamp custom field + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_restart_timestamp', + 'task_result': (action_restart_timestamp | string) + } | to_json }}" + payload_encoding: "string" + changed_when: false + delegate_to: localhost + when: + - is_run_by_effective == 'scheduler' + - action_restart_timestamp is defined + + - name: Remove bootenv lock marker after success + ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'" + changed_when: true + + rescue: + - name: Remove bootenv lock marker after failure + ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'" + changed_when: true + ignore_errors: true + + - name: Build failure task name and detail + ansible.builtin.set_fact: + fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}" + fail_detail_raw: >- + {{ ansible_failed_result.msg + | default(ansible_failed_result.stderr) + | default(ansible_failed_result.stdout) + | default('no additional error output') + | trim }} + + - name: Build failure summary text + ansible.builtin.set_fact: + fail_summary: >- + Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }} + + - name: Truncate failure summary to ~400 chars + ansible.builtin.set_fact: + fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}" + + - name: Build control queue payload for failure journal + ansible.builtin.set_fact: + journal_failure_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "{{ fail_summary_short }}" + + - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API + ansible.builtin.pause: + seconds: 1 + delegate_to: localhost + changed_when: false + + - name: Publish failure journal to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_failure_payload | to_json }}" + payload_encoding: "string" + register: rmq_journal_fail_resp + changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_fail_resp.status != 200) or + (rmq_journal_fail_resp.json is not defined) or + (not (rmq_journal_fail_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Scheduler | Publish action_state failed (bootenv failure) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_state', + 'task_result': 'failed' + } | to_json }}" + payload_encoding: "string" + changed_when: false + delegate_to: localhost + when: is_run_by_effective == 'scheduler' + + - name: Scheduler | Debug action_state failed published (bootenv failure) + ansible.builtin.debug: + msg: "scheduler-run detected; published action_state=failed (bootenv failure)" + when: is_run_by_effective == 'scheduler'