diff --git a/files/ansible-playbooks/update-indoor.yml b/files/ansible-playbooks/update-indoor.yml index 1ff1ace..44ea408 100644 --- a/files/ansible-playbooks/update-indoor.yml +++ b/files/ansible-playbooks/update-indoor.yml @@ -50,12 +50,7 @@ control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" - # --- TEMP: stall-test knobs (remove later) --- - aic_stall_test: true # enable/disable the simulated stall on DEV2 - aic_stall_sleep: 1000 # seconds to sleep on DEV2 (simulate hang) - aic_stall_async: 10 # async timeout (seconds) for the sleep job on DEV2 - aic_stall_poll: 2 # poll interval (seconds) while watching the job - + pre_tasks: # ------------------------------- Hostname sanity DEV1 ------------------------------- - name: Read DEV1 hostname (busybox-safe) @@ -461,63 +456,6 @@ _blocked: true _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - # test test test - - # === TEMP: STALL TEST ON DEV2 (BEGIN) ======================================== - # Enable with: -e aic_stall_test=true - # Optional knobs: -e aic_stall_sleep=1000 -e aic_stall_async=10 -e aic_stall_poll=2 - - - name: TEMP | Set stall test toggles from env/defaults - delegate_to: localhost - ansible.builtin.set_fact: - aic_stall_test: "{{ (aic_stall_test | default(lookup('env','AIC_STALL_TEST') | default('false', true))) | bool }}" - aic_stall_sleep: "{{ (aic_stall_sleep | default(lookup('env','AIC_STALL_SLEEP') | default(1000, true))) | int }}" - aic_stall_async: "{{ (aic_stall_async | default(lookup('env','AIC_STALL_ASYNC') | default(10, true))) | int }}" - aic_stall_poll: "{{ (aic_stall_poll | default(lookup('env','AIC_STALL_POLL') | default(2, true))) | int }}" - - - name: TEMP | Announce stall test settings - when: aic_stall_test - delegate_to: localhost - ansible.builtin.debug: - msg: - - "STALL TEST ENABLED" - - "sleep={{ aic_stall_sleep }}s async_cap={{ aic_stall_async }}s poll={{ aic_stall_poll }}s" - - - name: TEMP | Simulate a stuck operation on DEV2 via tunnel (async-capped) - when: aic_stall_test - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sh -c 'date; echo STALL-TEST-START; sleep {{ aic_stall_sleep }}; echo STALL-TEST-END; date'" - args: - executable: /bin/bash - async: "{{ aic_stall_async }}" - poll: "{{ aic_stall_poll }}" - register: stall_test_result - changed_when: false - failed_when: false # capture outcome; don't fail the whole play - - - name: TEMP | Show stall test result (expect timeout-ish when sleep > async) - when: aic_stall_test - delegate_to: localhost - ansible.builtin.debug: - var: stall_test_result - - - name: TEMP | Exit early after stall test (no device actions beyond this point) - when: aic_stall_test - ansible.builtin.meta: end_play - # === TEMP: STALL TEST ON DEV2 (END) ========================================== - - - # test test test - # ---------------------------- DEV2 version firmux primary check ---------------------------- - name: Read DEV2 /usr/lib/release/firmux (if present) when: dev2_passfile_used != "NONE" @@ -856,25 +794,102 @@ - name: Upgrade write and bank flip on DEV2 (guarded by soft-block) when: not (_blocked | default(false)) block: - - name: Write image to inactive bank on DEV2 (update -w) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') + # --- supervised async write with stall handling --- + - name: Write image to inactive bank on DEV2 (update -w, supervised async) + delegate_to: localhost + vars: + _write_async_cap: 600 # 10 minutes max runtime (tune if needed) + _write_poll: 20 # poll every 20 seconds + block: + - name: Launch update -w on DEV2 (async) + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + async: "{{ _write_async_cap }}" + poll: "{{ _write_poll }}" + register: dev2_up_write + changed_when: true + failed_when: false + + - name: Evaluate async result + ansible.builtin.set_fact: + _updatew_stalled: >- + {{ (dev2_up_write.msg is defined) and + ('did not complete' in dev2_up_write.msg | lower) }} + _updatew_success: >- + {{ (dev2_up_write.stdout | default('')) is search('update is complete') }} + + - name: Journal stall if update -w timed out + when: _updatew_stalled | bool + ansible.builtin.set_fact: + journal_updatew_stalled: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "update -w has stalled. Rebooting device; please schedule another upgrade." + + - name: Publish stall journal to control queue + when: journal_updatew_stalled is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_updatew_stalled | to_json }}" + payload_encoding: "string" + register: rmq_stall_journal_resp + changed_when: (rmq_stall_journal_resp.json is defined) and (rmq_stall_journal_resp.json.routed | default(false) | bool) + failed_when: false + + - name: Schedule immediate reboot (20s delay) after stall + when: _updatew_stalled | bool + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "nohup /sbin/reboot -d 20 >/dev/null 2>&1 &" + args: + executable: /bin/bash + changed_when: true + ignore_errors: true + + - name: Stop play if update -w stalled + when: _updatew_stalled | bool + ansible.builtin.meta: end_play + + - name: Fail play if update -w completed but invalid output + when: (not _updatew_stalled) and (not _updatew_success) + ansible.builtin.fail: + msg: "update -w finished but did not report 'update is complete'" + + # --- continue normal path if not stalled --- - name: Read current active partition on DEV2 (before flip) + when: not _updatew_stalled | bool delegate_to: localhost ansible.builtin.shell: | set -e @@ -893,12 +908,12 @@ failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - name: Determine new active value for DEV2 (flip 1↔2) - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + when: not _updatew_stalled | bool and dev2_active_before.stdout is defined ansible.builtin.set_fact: dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - name: Set fw_setenv active={{ dev2_new_active }} on DEV2 - when: dev2_new_active is defined + when: not _updatew_stalled | bool and dev2_new_active is defined delegate_to: localhost ansible.builtin.shell: | set -e @@ -916,7 +931,7 @@ changed_when: true - name: Verify active partition flipped on DEV2 (read back) - when: dev2_new_active is defined + when: not _updatew_stalled | bool and dev2_new_active is defined delegate_to: localhost ansible.builtin.shell: | set -e @@ -935,6 +950,7 @@ failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - name: Create specific prep marker on DEV2 for this image + when: not _updatew_stalled | bool delegate_to: localhost ansible.builtin.shell: | set -e @@ -951,6 +967,8 @@ register: dev2_marker_write changed_when: true ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- - name: Schedule DEV2 reboot after computed delay (seconds)