pavel/ansible-worker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1001

Browse Source
This commit is contained in:
pavel
2025-10-23 10:01:08 +03:00
parent 7b10ddfc6d
commit d0a9a9211b
1 changed files with 101 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
files/ansible-playbooks/update-indoor.yml
View File

@@ -50,11 +50,6 @@
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
# --- TEMP: stall-test knobs (remove later) ---
aic_stall_test: true # enable/disable the simulated stall on DEV2
aic_stall_sleep: 1000 # seconds to sleep on DEV2 (simulate hang)
aic_stall_async: 10 # async timeout (seconds) for the sleep job on DEV2
aic_stall_poll: 2 # poll interval (seconds) while watching the job
pre_tasks:
# ------------------------------- Hostname sanity DEV1 -------------------------------
@@ -461,63 +456,6 @@
_blocked: true
_journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
# test test test
# === TEMP: STALL TEST ON DEV2 (BEGIN) ========================================
# Enable with: -e aic_stall_test=true
# Optional knobs: -e aic_stall_sleep=1000 -e aic_stall_async=10 -e aic_stall_poll=2
- name: TEMP | Set stall test toggles from env/defaults
delegate_to: localhost
ansible.builtin.set_fact:
aic_stall_test: "{{ (aic_stall_test | default(lookup('env','AIC_STALL_TEST') | default('false', true))) | bool }}"
aic_stall_sleep: "{{ (aic_stall_sleep | default(lookup('env','AIC_STALL_SLEEP') | default(1000, true))) | int }}"
aic_stall_async: "{{ (aic_stall_async | default(lookup('env','AIC_STALL_ASYNC') | default(10, true))) | int }}"
aic_stall_poll: "{{ (aic_stall_poll | default(lookup('env','AIC_STALL_POLL') | default(2, true))) | int }}"
- name: TEMP | Announce stall test settings
when: aic_stall_test
delegate_to: localhost
ansible.builtin.debug:
msg:
- "STALL TEST ENABLED"
- "sleep={{ aic_stall_sleep }}s async_cap={{ aic_stall_async }}s poll={{ aic_stall_poll }}s"
- name: TEMP | Simulate a stuck operation on DEV2 via tunnel (async-capped)
when: aic_stall_test
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"sh -c 'date; echo STALL-TEST-START; sleep {{ aic_stall_sleep }}; echo STALL-TEST-END; date'"
args:
executable: /bin/bash
async: "{{ aic_stall_async }}"
poll: "{{ aic_stall_poll }}"
register: stall_test_result
changed_when: false
failed_when: false # capture outcome; don't fail the whole play
- name: TEMP | Show stall test result (expect timeout-ish when sleep > async)
when: aic_stall_test
delegate_to: localhost
ansible.builtin.debug:
var: stall_test_result
- name: TEMP | Exit early after stall test (no device actions beyond this point)
when: aic_stall_test
ansible.builtin.meta: end_play
# === TEMP: STALL TEST ON DEV2 (END) ==========================================
# test test test
# ---------------------------- DEV2 version firmux primary check ----------------------------
- name: Read DEV2 /usr/lib/release/firmux (if present)
when: dev2_passfile_used != "NONE"
@@ -856,8 +794,15 @@
- name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
when: not (_blocked | default(false))
block:
- name: Write image to inactive bank on DEV2 (update -w)
# --- supervised async write with stall handling ---
- name: Write image to inactive bank on DEV2 (update -w, supervised async)
delegate_to: localhost
vars:
_write_async_cap: 600 # 10 minutes max runtime (tune if needed)
_write_poll: 20 # poll every 20 seconds
block:
- name: Launch update -w on DEV2 (async)
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
@@ -870,11 +815,81 @@
"update -w '{{ dev2_image_path }}'"
args:
executable: /bin/bash
async: "{{ _write_async_cap }}"
poll: "{{ _write_poll }}"
register: dev2_up_write
changed_when: true
failed_when: dev2_up_write.stdout is not search('update is complete')
failed_when: false
- name: Evaluate async result
ansible.builtin.set_fact:
_updatew_stalled: >-
{{ (dev2_up_write.msg is defined) and
('did not complete' in dev2_up_write.msg | lower) }}
_updatew_success: >-
{{ (dev2_up_write.stdout | default('')) is search('update is complete') }}
- name: Journal stall if update -w timed out
when: _updatew_stalled | bool
ansible.builtin.set_fact:
journal_updatew_stalled:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: "update -w has stalled. Rebooting device; please schedule another upgrade."
- name: Publish stall journal to control queue
when: journal_updatew_stalled is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_updatew_stalled | to_json }}"
payload_encoding: "string"
register: rmq_stall_journal_resp
changed_when: (rmq_stall_journal_resp.json is defined) and (rmq_stall_journal_resp.json.routed | default(false) | bool)
failed_when: false
- name: Schedule immediate reboot (20s delay) after stall
when: _updatew_stalled | bool
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"nohup /sbin/reboot -d 20 >/dev/null 2>&1 &"
args:
executable: /bin/bash
changed_when: true
ignore_errors: true
- name: Stop play if update -w stalled
when: _updatew_stalled | bool
ansible.builtin.meta: end_play
- name: Fail play if update -w completed but invalid output
when: (not _updatew_stalled) and (not _updatew_success)
ansible.builtin.fail:
msg: "update -w finished but did not report 'update is complete'"
# --- continue normal path if not stalled ---
- name: Read current active partition on DEV2 (before flip)
when: not _updatew_stalled | bool
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -893,12 +908,12 @@
failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
- name: Determine new active value for DEV2 (flip 1↔2)
when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
when: not _updatew_stalled | bool and dev2_active_before.stdout is defined
ansible.builtin.set_fact:
dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
- name: Set fw_setenv active={{ dev2_new_active }} on DEV2
when: dev2_new_active is defined
when: not _updatew_stalled | bool and dev2_new_active is defined
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -916,7 +931,7 @@
changed_when: true
- name: Verify active partition flipped on DEV2 (read back)
when: dev2_new_active is defined
when: not _updatew_stalled | bool and dev2_new_active is defined
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -935,6 +950,7 @@
failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
- name: Create specific prep marker on DEV2 for this image
when: not _updatew_stalled | bool
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -952,6 +968,8 @@
changed_when: true
ignore_errors: true
# ---------------------------- Reboot scheduling (normalized) ----------------------------
- name: Schedule DEV2 reboot after computed delay (seconds)
when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))