792 lines
34 KiB
YAML
792 lines
34 KiB
YAML
- name: Fix outdoor bootenv safely (no Python on target)
|
|
hosts: all
|
|
gather_facts: no
|
|
|
|
vars:
|
|
# RabbitMQ (pull from env if provided)
|
|
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
|
|
|
# Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
|
|
is_run_by_effective: "{{ is_run_by | default('manual') }}"
|
|
|
|
# Future post-bootenv check scheduling (kept disabled for now)
|
|
afterbootenv_hold_exchange: "{{ lookup('env','AFTERBOOTENV_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
|
|
afterbootenv_routing_key: "{{ lookup('env','AFTERBOOTENV_ROUTING_KEY') | default('deviceconfig', true) }}"
|
|
afterbootenv_hold_queue: "{{ lookup('env','AFTERBOOTENV_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
|
|
afterbootenv_check_enabled: false
|
|
|
|
bootenv_filename: "fox100_bootenv.bin"
|
|
bootenv_sha256: "324337e20b0a2d8048c359bfa2a1b8dffd6b1a28eab260143dd895ca39c034aa"
|
|
bootenv_path: "/tmp/{{ bootenv_filename }}"
|
|
bootenv_mtd_device: "/dev/mtdblock8"
|
|
bootenv_expected_size: "65536"
|
|
|
|
# Helper computed vars
|
|
bootenv_name: "{{ bootenv_filename | regex_replace('\\.bin$', '') }}"
|
|
bootenv_marker: "/tmp/bootenv_fixed_{{ bootenv_name }}"
|
|
bootenv_lock_marker: "/tmp/bootenv_fix_inprogress_{{ bootenv_name }}"
|
|
firmware_guard_marker: "/tmp/prepared_for_{{ bootenv_name }}"
|
|
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
|
|
|
tasks:
|
|
|
|
- name: Hostname preflight
|
|
block:
|
|
- name: Read remote HOSTNAME
|
|
ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
|
|
register: host_env
|
|
changed_when: false
|
|
|
|
- name: Debug hostnames
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "remote_hostname={{ host_env.stdout | trim }}"
|
|
- "inventory_hostname={{ inventory_hostname }}"
|
|
|
|
- name: Stop if connected hostname differs from inventory
|
|
ansible.builtin.fail:
|
|
msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
|
|
when: (host_env.stdout | trim) != inventory_hostname
|
|
|
|
rescue:
|
|
- name: Build failure task name and detail (hostname preflight)
|
|
ansible.builtin.set_fact:
|
|
fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
|
|
fail_detail_raw: >-
|
|
{{ ansible_failed_result.msg
|
|
| default(ansible_failed_result.stderr)
|
|
| default(ansible_failed_result.stdout)
|
|
| default('no additional error output')
|
|
| trim }}
|
|
|
|
- name: Build failure summary text (hostname preflight)
|
|
ansible.builtin.set_fact:
|
|
fail_summary: >-
|
|
Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
|
|
|
|
- name: Truncate failure summary to ~400 chars (hostname preflight)
|
|
ansible.builtin.set_fact:
|
|
fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
|
|
|
|
- name: Build control queue payload for failure journal (hostname preflight)
|
|
ansible.builtin.set_fact:
|
|
journal_failure_payload_pre:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: "{{ fail_summary_short }}"
|
|
|
|
- name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
|
|
ansible.builtin.pause:
|
|
seconds: 1
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
|
|
- name: Publish failure journal to control queue (hostname preflight)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_failure_payload_pre | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_journal_pre_resp
|
|
changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
|
|
failed_when: >
|
|
(rmq_journal_pre_resp.status != 200) or
|
|
(rmq_journal_pre_resp.json is not defined) or
|
|
(not (rmq_journal_pre_resp.json.routed | default(false) | bool))
|
|
delegate_to: localhost
|
|
|
|
- name: Scheduler | Publish action_state failed (hostname preflight)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'action_state',
|
|
'task_result': 'failed'
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when: is_run_by_effective == 'scheduler'
|
|
|
|
- name: Scheduler | Debug action_state failed published (hostname preflight)
|
|
ansible.builtin.debug:
|
|
msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
|
|
when: is_run_by_effective == 'scheduler'
|
|
|
|
- name: Stop play after hostname preflight failure
|
|
ansible.builtin.meta: end_play
|
|
|
|
- name: Scheduler context | Debug is_run_by
|
|
ansible.builtin.debug:
|
|
msg: "is_run_by={{ is_run_by_effective }}"
|
|
|
|
- name: Scheduler | Publish action_state inprogress (scheduler-run)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'action_state',
|
|
'task_result': 'inprogress'
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when: is_run_by_effective == 'scheduler'
|
|
|
|
- name: Scheduler | Debug action_state inprogress published
|
|
ansible.builtin.debug:
|
|
msg: "scheduler-run detected; published action_state=inprogress"
|
|
when: is_run_by_effective == 'scheduler'
|
|
|
|
- name: Check if any prepared marker exists
|
|
ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
|
|
register: prep_scan
|
|
changed_when: false
|
|
|
|
- name: Debug marker presence
|
|
ansible.builtin.debug:
|
|
msg: "prepared_marker={{ prep_scan.stdout | trim }}"
|
|
|
|
- name: Journal preparation already present, skipping bootenv steps
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "journal_add",
|
|
"task_result": (
|
|
"Bootenv fix cancelled because a prepared marker is already present; " ~
|
|
"expected marker " ~ firmware_guard_marker ~ ". Skipping bootenv steps."
|
|
)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_journal_prep_present
|
|
changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
|
|
failed_when: >
|
|
(rmq_journal_prep_present.status != 200) or
|
|
(rmq_journal_prep_present.json is not defined) or
|
|
(not (rmq_journal_prep_present.json.routed | default(false) | bool))
|
|
delegate_to: localhost
|
|
when: (prep_scan.stdout | trim) == 'PRESENT'
|
|
|
|
- name: End play for this host (already prepared)
|
|
ansible.builtin.meta: end_host
|
|
when: (prep_scan.stdout | trim) == 'PRESENT'
|
|
|
|
- name: Check if bootenv lock marker already exists
|
|
ansible.builtin.raw: "{{ pathprefix }} [ -e '{{ bootenv_lock_marker }}' ] && echo PRESENT || echo ABSENT"
|
|
register: bootenv_lock_scan
|
|
changed_when: false
|
|
|
|
- name: Debug bootenv lock marker presence
|
|
ansible.builtin.debug:
|
|
msg: "bootenv_lock_marker={{ bootenv_lock_scan.stdout | trim }}"
|
|
|
|
- name: Journal bootenv fix already in progress, skipping
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "journal_add",
|
|
"task_result": (
|
|
"Bootenv fix cancelled because lock marker already exists; " ~
|
|
"marker " ~ bootenv_lock_marker ~ ". Another process may be running."
|
|
)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_journal_lock_present
|
|
changed_when: (rmq_journal_lock_present.json is defined) and (rmq_journal_lock_present.json.routed | default(false) | bool)
|
|
failed_when: >
|
|
(rmq_journal_lock_present.status != 200) or
|
|
(rmq_journal_lock_present.json is not defined) or
|
|
(not (rmq_journal_lock_present.json.routed | default(false) | bool))
|
|
delegate_to: localhost
|
|
when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
|
|
|
|
- name: End play for this host (bootenv lock already present)
|
|
ansible.builtin.meta: end_host
|
|
when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
|
|
|
|
- name: Create bootenv lock marker
|
|
ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_lock_marker }}'"
|
|
changed_when: true
|
|
|
|
- name: Bootenv fix main block
|
|
block:
|
|
- name: Count fw_printenv lines before bootenv write
|
|
ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
|
|
register: env_line_count_before
|
|
changed_when: false
|
|
|
|
- name: Debug fw_printenv line count before bootenv write
|
|
ansible.builtin.debug:
|
|
msg: "fw_printenv_lines_before={{ env_line_count_before.stdout | trim }}"
|
|
|
|
- name: Note if bootloader environment looks invalid before bootenv write (<7 lines)
|
|
ansible.builtin.debug:
|
|
msg: "Proceeding with bootenv repair even though fw_printenv returned only {{ env_line_count_before.stdout | trim }} lines (<7) before bootenv write."
|
|
when: (env_line_count_before.stdout | trim | int) < 7
|
|
|
|
- name: Check if bootenv image is already on the device
|
|
ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ bootenv_path }}' ] && echo OK || echo MISSING"
|
|
register: bootenv_exists
|
|
changed_when: false
|
|
|
|
- name: Upload bootenv to /tmp via scp (controller-side)
|
|
ansible.builtin.shell: |
|
|
set -e
|
|
SRC='{{ bootenv_src_local | default("/opt/containers/ansible-worker/files/fox100_bootenv.bin") }}'
|
|
DST_USER='{{ ansible_user | default("root") }}'
|
|
DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
|
|
test -f "$SRC"
|
|
sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
"$SRC" "${DST_USER}@${DST_HOST}:{{ bootenv_path }}"
|
|
delegate_to: localhost
|
|
when: bootenv_exists.stdout is not defined or (bootenv_exists.stdout | trim) != 'OK'
|
|
changed_when: true
|
|
|
|
- name: Re-check bootenv presence after optional upload
|
|
ansible.builtin.raw: "{{ pathprefix }} test -f '{{ bootenv_path }}' && echo OK || echo MISSING"
|
|
register: bootenv_exists2
|
|
changed_when: false
|
|
failed_when: (bootenv_exists2.stdout | trim) != 'OK'
|
|
|
|
- name: Compute sha256 of the uploaded bootenv image
|
|
ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ bootenv_path }}' | awk '{print $1}'"
|
|
register: bootenv_sha_out
|
|
changed_when: false
|
|
|
|
- name: Verify bootenv sha256 matches expected
|
|
ansible.builtin.fail:
|
|
msg: "SHA256 mismatch for {{ bootenv_path }}. Got {{ bootenv_sha_out.stdout | trim }}, expected {{ bootenv_sha256 }}"
|
|
when: (bootenv_sha_out.stdout | trim) != (bootenv_sha256 | trim)
|
|
|
|
- name: Bootenv sha256 verification debug
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "sha256sum is: {{ bootenv_sha_out.stdout | trim }}"
|
|
|
|
- name: Read bootenv image size on target
|
|
ansible.builtin.raw: "{{ pathprefix }} wc -c < '{{ bootenv_path }}'"
|
|
register: bootenv_size_out
|
|
changed_when: false
|
|
|
|
- name: Verify bootenv image size matches expected
|
|
ansible.builtin.fail:
|
|
msg: "Bootenv size mismatch for {{ bootenv_path }}. Got {{ bootenv_size_out.stdout | trim }}, expected {{ bootenv_expected_size }}"
|
|
when: (bootenv_size_out.stdout | trim) != (bootenv_expected_size | string | trim)
|
|
|
|
- name: Bootenv size verification debug
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "bootenv_size={{ bootenv_size_out.stdout | trim }}"
|
|
|
|
- name: Write bootenv to flash with dd
|
|
ansible.builtin.raw: "{{ pathprefix }} dd if='{{ bootenv_path }}' of={{ bootenv_mtd_device }}"
|
|
register: bootenv_dd
|
|
changed_when: true
|
|
failed_when: bootenv_dd.rc != 0
|
|
|
|
- name: Debug dd output (bootenv)
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "dd.rc={{ bootenv_dd.rc | default('NA') }}"
|
|
- "dd.stdout={{ (bootenv_dd.stdout | default('')) | trim }}"
|
|
- "dd.stderr={{ (bootenv_dd.stderr | default('')) | trim }}"
|
|
|
|
- name: Run sync after bootenv write
|
|
ansible.builtin.raw: "{{ pathprefix }} sync"
|
|
register: bootenv_sync
|
|
changed_when: true
|
|
failed_when: bootenv_sync.rc != 0
|
|
|
|
- name: Debug sync output (bootenv)
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "sync.rc={{ bootenv_sync.rc | default('NA') }}"
|
|
- "sync.stdout={{ (bootenv_sync.stdout | default('')) | trim }}"
|
|
- "sync.stderr={{ (bootenv_sync.stderr | default('')) | trim }}"
|
|
|
|
- name: Set bootenv write success flag
|
|
ansible.builtin.set_fact:
|
|
_bootenv_write_success: "{{ (bootenv_dd.rc | default(1)) == 0 and (bootenv_sync.rc | default(1)) == 0 }}"
|
|
|
|
- name: Create prepared marker
|
|
ansible.builtin.raw: "{{ pathprefix }} touch '{{ firmware_guard_marker }}'"
|
|
changed_when: true
|
|
|
|
- name: Create bootenv marker
|
|
ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_marker }}'"
|
|
changed_when: true
|
|
|
|
- name: Build control queue payload (progress & target bootenv)
|
|
ansible.builtin.set_fact:
|
|
nbq2_payload_obj:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "bootenv_outdoor_wo_restart"
|
|
task_result: "waiting_restart"
|
|
task_add1: "{{ bootenv_filename }}"
|
|
when: _bootenv_write_success | bool
|
|
|
|
- name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
|
|
ansible.builtin.pause:
|
|
seconds: 1
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
|
|
- name: Publish message to control queue via RabbitMQ HTTP API
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ nbq2_payload_obj | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_resp
|
|
changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
|
|
failed_when: >
|
|
(rmq_resp.status != 200) or
|
|
(rmq_resp.json is not defined) or
|
|
(rmq_resp.json.routed | default(false) | bool == false)
|
|
delegate_to: localhost
|
|
when: nbq2_payload_obj is defined
|
|
|
|
- name: Log control queue publish result
|
|
ansible.builtin.debug:
|
|
var: rmq_resp.json
|
|
when: rmq_resp is defined
|
|
|
|
- name: Summary
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "Bootenv target file: {{ bootenv_filename }}"
|
|
- "Bootenv SHA256: OK ({{ bootenv_sha_out.stdout | trim }})"
|
|
- "Bootenv size: {{ bootenv_size_out.stdout | trim }} bytes"
|
|
- "fw_printenv lines before write: {{ env_line_count_before.stdout | trim }}"
|
|
- "dd: OK"
|
|
- "sync: OK"
|
|
- "Marker: {{ bootenv_marker }}"
|
|
- "Guard marker: {{ firmware_guard_marker }}"
|
|
|
|
- name: Compute reboot delay in seconds (default immediate when rebootin missing)
|
|
ansible.builtin.set_fact:
|
|
reboot_seconds: "{{ (rebootin | default(0) | int) * 3600 }}"
|
|
when:
|
|
- nbq2_payload_obj is defined
|
|
|
|
- name: Schedule delayed reboot on device (HUP-safe)
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
|
|
changed_when: true
|
|
when:
|
|
- nbq2_payload_obj is defined
|
|
- reboot_seconds is defined
|
|
|
|
- name: Build control queue payload for success journal
|
|
ansible.builtin.set_fact:
|
|
journal_success_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
Bootenv fix complete using {{ bootenv_filename }} written to {{ bootenv_mtd_device }}.
|
|
Original fw_printenv line count was {{ env_line_count_before.stdout | trim }}.
|
|
SHA256 OK {{ bootenv_sha_out.stdout | trim }}.
|
|
Size {{ bootenv_size_out.stdout | trim }} bytes.
|
|
Marker {{ bootenv_marker }}.
|
|
{{
|
|
('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate bootenv change.')
|
|
if (rebootin is defined)
|
|
else 'Waiting for restart to activate bootenv change.'
|
|
}}
|
|
when: nbq2_payload_obj is defined
|
|
|
|
- name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
|
|
ansible.builtin.pause:
|
|
seconds: 1
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
|
|
- name: Publish success journal to control queue via RabbitMQ HTTP API
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_success_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_journal_success_resp
|
|
changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
|
|
failed_when: >
|
|
(rmq_journal_success_resp.status != 200) or
|
|
(rmq_journal_success_resp.json is not defined) or
|
|
(not (rmq_journal_success_resp.json.routed | default(false) | bool))
|
|
delegate_to: localhost
|
|
|
|
- name: Init after-bootenv scheduling vars
|
|
ansible.builtin.set_fact:
|
|
ab_attempt: 1
|
|
ab_max_attempts: 3
|
|
ab_delay_sec: >-
|
|
{{
|
|
(
|
|
(hostvars[inventory_hostname].reboot_seconds | default(0) | int)
|
|
+ 300
|
|
)
|
|
if (hostvars[inventory_hostname].reboot_seconds is defined)
|
|
else 300
|
|
}}
|
|
when: nbq2_payload_obj is defined
|
|
delegate_to: localhost
|
|
|
|
- name: Generate correlation ID and original emitted timestamp
|
|
ansible.builtin.set_fact:
|
|
ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
|
|
ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
|
|
when: nbq2_payload_obj is defined
|
|
delegate_to: localhost
|
|
|
|
- name: Build after-bootenv check payload (attempt 1)
|
|
ansible.builtin.set_fact:
|
|
ab_attempt: 1
|
|
ab_delay_sec: "{{ ab_delay_sec | default(300) }}"
|
|
ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
|
|
ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
|
|
afterbootenv_payload:
|
|
task_name: "afterbootenv_outdoor_check"
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
target_bootenv_file: "{{ bootenv_filename }}"
|
|
original_fw_printenv_lines: "{{ env_line_count_before.stdout | trim }}"
|
|
attempt: "{{ ab_attempt | default(1) }}"
|
|
max_attempts: "{{ ab_max_attempts | default(3) }}"
|
|
current_delay_sec: "{{ ab_delay_sec | default(300) }}"
|
|
correlation_id: "{{ ab_correlation_id }}"
|
|
original_emitted_at: "{{ ab_original_emitted_at }}"
|
|
schema_version: 1
|
|
is_run_by: "{{ is_run_by_effective }}"
|
|
when: nbq2_payload_obj is defined
|
|
delegate_to: localhost
|
|
|
|
- name: Debug x-delay about to be sent (ms)
|
|
ansible.builtin.debug:
|
|
msg: "x-delay(ms) = {{ (ab_delay_sec | int) * 1000 }}"
|
|
when: afterbootenv_payload is defined
|
|
delegate_to: localhost
|
|
|
|
- name: DEBUG after-bootenv payload and timing
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "afterbootenv_payload={{ afterbootenv_payload | to_nice_json }}"
|
|
- "ab_delay_sec={{ ab_delay_sec }}"
|
|
- "afterbootenv_check_enabled={{ afterbootenv_check_enabled }}"
|
|
- "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
|
|
|
|
- name: Bandaid | Pause before Publish delayed after-bootenv check to holding exchange
|
|
ansible.builtin.pause:
|
|
seconds: 1
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
when:
|
|
- afterbootenv_payload is defined
|
|
- afterbootenv_check_enabled | bool
|
|
|
|
- name: Publish delayed after-bootenv check to holding exchange
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
headers:
|
|
x-delay: "{{ (ab_delay_sec | int) * 1000 }}"
|
|
routing_key: "{{ afterbootenv_routing_key }}"
|
|
payload: "{{ afterbootenv_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_afterbootenv_resp
|
|
changed_when: (rmq_afterbootenv_resp.json is defined) and (rmq_afterbootenv_resp.json.routed | default(false) | bool)
|
|
failed_when: >
|
|
(rmq_afterbootenv_resp.status != 200) or
|
|
(rmq_afterbootenv_resp.json is not defined)
|
|
when:
|
|
- afterbootenv_payload is defined
|
|
- afterbootenv_check_enabled | bool
|
|
delegate_to: localhost
|
|
|
|
- name: Scheduler | Publish action_state waiting (success)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'action_state',
|
|
'task_result': 'waiting'
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when: is_run_by_effective == 'scheduler'
|
|
|
|
- name: Scheduler | Compute action_restart_timestamp (unix seconds) when rebootin == 0
|
|
ansible.builtin.set_fact:
|
|
action_restart_timestamp: "{{ lookup('pipe','date -u +%s') | int }}"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when:
|
|
- is_run_by_effective == 'scheduler'
|
|
- reboot_seconds is defined
|
|
- (reboot_seconds | int) == 0
|
|
|
|
- name: Scheduler | Compute action_restart_timestamp (unix seconds) when reboot scheduled
|
|
ansible.builtin.set_fact:
|
|
action_restart_timestamp: "{{ (lookup('pipe','date -u +%s') | int) + (reboot_seconds | int) }}"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when:
|
|
- is_run_by_effective == 'scheduler'
|
|
- reboot_seconds is defined
|
|
- (reboot_seconds | int) > 0
|
|
|
|
- name: Scheduler | Debug computed action_restart_timestamp
|
|
ansible.builtin.debug:
|
|
msg: "action_restart_timestamp={{ action_restart_timestamp }} (reboot_seconds={{ reboot_seconds | int }})"
|
|
when:
|
|
- is_run_by_effective == 'scheduler'
|
|
- action_restart_timestamp is defined
|
|
delegate_to: localhost
|
|
|
|
- name: Scheduler | Publish action_restart_timestamp custom field
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'action_restart_timestamp',
|
|
'task_result': (action_restart_timestamp | string)
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when:
|
|
- is_run_by_effective == 'scheduler'
|
|
- action_restart_timestamp is defined
|
|
|
|
- name: Remove bootenv lock marker after success
|
|
ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
|
|
changed_when: true
|
|
|
|
rescue:
|
|
- name: Remove bootenv lock marker after failure
|
|
ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
|
|
changed_when: true
|
|
ignore_errors: true
|
|
|
|
- name: Build failure task name and detail
|
|
ansible.builtin.set_fact:
|
|
fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
|
|
fail_detail_raw: >-
|
|
{{ ansible_failed_result.msg
|
|
| default(ansible_failed_result.stderr)
|
|
| default(ansible_failed_result.stdout)
|
|
| default('no additional error output')
|
|
| trim }}
|
|
|
|
- name: Build failure summary text
|
|
ansible.builtin.set_fact:
|
|
fail_summary: >-
|
|
Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
|
|
|
|
- name: Truncate failure summary to ~400 chars
|
|
ansible.builtin.set_fact:
|
|
fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
|
|
|
|
- name: Build control queue payload for failure journal
|
|
ansible.builtin.set_fact:
|
|
journal_failure_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: "{{ fail_summary_short }}"
|
|
|
|
- name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
|
|
ansible.builtin.pause:
|
|
seconds: 1
|
|
delegate_to: localhost
|
|
changed_when: false
|
|
|
|
- name: Publish failure journal to control queue via RabbitMQ HTTP API
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_failure_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_journal_fail_resp
|
|
changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
|
|
failed_when: >
|
|
(rmq_journal_fail_resp.status != 200) or
|
|
(rmq_journal_fail_resp.json is not defined) or
|
|
(not (rmq_journal_fail_resp.json.routed | default(false) | bool))
|
|
delegate_to: localhost
|
|
|
|
- name: Scheduler | Publish action_state failed (bootenv failure)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'action_state',
|
|
'task_result': 'failed'
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
when: is_run_by_effective == 'scheduler'
|
|
|
|
- name: Scheduler | Debug action_state failed published (bootenv failure)
|
|
ansible.builtin.debug:
|
|
msg: "scheduler-run detected; published action_state=failed (bootenv failure)"
|
|
when: is_run_by_effective == 'scheduler'
|