From e1653cf1b2b1290629c01ba31e4868c458bf67df Mon Sep 17 00:00:00 2001 From: pavel Date: Wed, 22 Oct 2025 21:35:07 +0300 Subject: [PATCH] 2135 --- .../afterupgrade_indoor_check.yml | 164 +++++++++--------- files/ansible-playbooks/update-indoor.yml | 27 +++ 2 files changed, 109 insertions(+), 82 deletions(-) diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml b/files/ansible-playbooks/afterupgrade_indoor_check.yml index feee69d..cd71618 100644 --- a/files/ansible-playbooks/afterupgrade_indoor_check.yml +++ b/files/ansible-playbooks/afterupgrade_indoor_check.yml @@ -1,5 +1,5 @@ --- -- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel) +- name: After upgrade verification for indoor DEV2 via DEV1 tunnel hosts: all gather_facts: no @@ -31,8 +31,8 @@ control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" tasks: - # ---- Normalize metadata (from delayed message) ---- - - name: Normalize after-upgrade metadata + # ---- Normalize metadata from delayed message ---- + - name: Normalize after upgrade metadata into facts delegate_to: localhost ansible.builtin.set_fact: attempt: "{{ (attempt | default(1)) | int }}" @@ -42,8 +42,8 @@ target_version: "{{ target_version | default('') }}" target_version_full: "{{ target_version | default('') }}" - # (NEW) Show what we received from the scheduler - - name: Debug received scheduler metadata + # Visibility of what scheduler sent + - name: Debug show received scheduler metadata delegate_to: localhost ansible.builtin.debug: msg: @@ -53,8 +53,8 @@ - "original_emitted_at={{ original_emitted_at }}" - "target_version_full={{ target_version_full }}" - # ---- Controller-side TCP probe to DEV1 ---- - - name: Check if TCP 22 on DEV1 is reachable + # ---- Controller side TCP probe to DEV1 ---- + - name: Probe DEV1 TCP 22 from controller delegate_to: localhost ansible.builtin.shell: | nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22 @@ -62,8 +62,8 @@ changed_when: false ignore_errors: true - # ---- If TCP down: journal + schedule next try or give up ---- - - name: Build failure journal TCP unreachable and mark retry + # ---- If TCP down publish journal and schedule next try or give up ---- + - name: Build journal payload for DEV1 TCP unreachable and mark retry when: nc_probe.rc != 0 delegate_to: localhost ansible.builtin.set_fact: @@ -76,7 +76,7 @@ Original={{ original_emitted_at }} Target='{{ target_version_full }}' _needs_retry: true - - name: Publish failure journal TCP unreachable + - name: Publish journal for DEV1 TCP unreachable when: nc_probe.rc != 0 delegate_to: localhost ansible.builtin.uri: @@ -96,7 +96,7 @@ register: rmq_j_tcp_fail changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) - - name: Compute next-attempt delay 10 minutes and counters + - name: Compute next attempt delay ten minutes and counters when: (_needs_retry | default(false)) | bool delegate_to: localhost ansible.builtin.set_fact: @@ -104,7 +104,7 @@ next_delay_sec: 600 next_delay_ms: 600000 - - name: Build final gave-up journal max attempts reached + - name: Build final gave up journal when max attempts reached when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) delegate_to: localhost ansible.builtin.set_fact: @@ -116,7 +116,7 @@ Exhausted attempts. Last error path=TCP. Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - - name: Publish final gave-up journal + - name: Publish final gave up journal after TCP failures when: journal_gaveup_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -136,11 +136,11 @@ register: rmq_j_gaveup changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) - - name: Stop host after final gave-up + - name: Stop host after final gave up due to TCP failure when: journal_gaveup_payload is defined ansible.builtin.meta: end_host - - name: Build delayed after-upgrade payload for next attempt 10m + - name: Build delayed payload for next attempt after upgrade ten minutes when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) delegate_to: localhost ansible.builtin.set_fact: @@ -155,7 +155,7 @@ current_delay_sec: "{{ next_delay_sec | int }}" schema_version: 1 - - name: Publish delayed next attempt to holding dead-letters to deviceconfig + - name: Publish delayed next attempt to holding exchange with TTL to deviceconfig when: delayed_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -183,8 +183,8 @@ ansible.builtin.meta: end_host - # ---------------- Step 1: Read DEV1 hostname and sanity ---------------- - - name: Read DEV1 hostname + # ---------------- Step 1 Read DEV1 hostname and sanity ---------------- + - name: Read DEV1 hostname from kernel procfs when: nc_probe.rc == 0 ansible.builtin.raw: > {{ pathprefix }} @@ -192,13 +192,13 @@ register: dev1_host_read changed_when: false - - name: Stop if DEV1 hostname mismatch + - name: Stop if DEV1 hostname does not match inventory hostname ansible.builtin.meta: end_host when: (dev1_host_read.stdout | trim | length > 0) and ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - # ---------------- Step 2: Setup temporary IP for reachability ---------------- - - name: Add temporary IP on DEV1 ignore if exists + # ---------------- Step 2 Setup temporary IP for reachability ---------------- + - name: Add temporary IP on DEV1 ignore if address already exists ansible.builtin.raw: > {{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} @@ -209,8 +209,8 @@ ('File exists' not in (add_ip.stdout | default(''))) and ('File exists' not in (add_ip.stderr | default(''))) - # ---------------- Step 3: Start SSH tunnel via DEV1 ---------------- - - name: Pick random free local port + # ---------------- Step 3 Start SSH tunnel via DEV1 ---------------- + - name: Pick random free local port for SSH tunnel delegate_to: localhost ansible.builtin.shell: | set -e @@ -222,22 +222,22 @@ register: pick_port changed_when: false - - name: Stop if no free local port + - name: Stop if no free local port was found ansible.builtin.meta: end_host when: (pick_port.stdout | trim | length) == 0 - - name: Record chosen local port and control dir + - name: Record chosen local port and create control directory delegate_to: localhost ansible.builtin.set_fact: _local_port: "{{ pick_port.stdout | trim }}" _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - name: Build tunnel control socket path + - name: Build SSH control socket path for tunnel delegate_to: localhost ansible.builtin.set_fact: _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - name: Start SSH ControlMaster tunnel via DEV1 + - name: Start SSH ControlMaster tunnel via DEV1 to DEV2 delegate_to: localhost ansible.builtin.shell: | set -e @@ -251,14 +251,14 @@ register: start_tunnel changed_when: true - # --- settle + check ControlMaster + TCP probe --- - - name: Small delay for tunnel to settle + # --- settle and check ControlMaster and TCP probe --- + - name: Pause briefly to let SSH tunnel settle delegate_to: localhost ansible.builtin.wait_for: timeout: 1 changed_when: false - - name: Verify tunnel master running ssh O check + - name: Verify tunnel ControlMaster is running with ssh O check delegate_to: localhost ansible.builtin.shell: | set -e @@ -267,14 +267,14 @@ register: tun_check changed_when: false - - name: Debug tunnel check + - name: Debug show tunnel master check result delegate_to: localhost ansible.builtin.debug: msg: - "tunnel_check.rc={{ tun_check.rc }}" - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - name: Sanity confirm tunnel TCP reachability to DEV2 + - name: Probe TCP reachability through tunnel to DEV2 delegate_to: localhost ansible.builtin.shell: | set -e @@ -283,7 +283,7 @@ changed_when: false ignore_errors: true - - name: Debug reachability result + - name: Debug show reachability probe result through tunnel delegate_to: localhost ansible.builtin.debug: msg: @@ -291,12 +291,12 @@ - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - name: Stop if tunnel TCP check failed + - name: Stop if tunnel reachability check failed ansible.builtin.meta: end_host when: nc_probe.rc != 0 - # ---------------- Step 4: Determine working password for DEV2 ---------------- - - name: Try both passfiles for DEV2 + # ---------------- Step 4 Determine working password for DEV2 ---------------- + - name: Try provided passfiles to authenticate to DEV2 delegate_to: localhost ansible.builtin.shell: | for f in {{ dev2_passfiles | join(' ') }}; do @@ -308,23 +308,23 @@ register: dev2_passfile_try changed_when: false - - name: Save selected DEV2 passfile + - name: Save selected DEV2 passfile for subsequent commands delegate_to: localhost ansible.builtin.set_fact: dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}" changed_when: false - - name: Debug selected DEV2 passfile + - name: Debug show which DEV2 passfile was selected delegate_to: localhost ansible.builtin.debug: msg: "dev2_passfile_used={{ dev2_passfile_used }}" - - name: Stop if no valid passfile found + - name: Stop if no valid DEV2 passfile was found ansible.builtin.meta: end_host when: dev2_passfile_used == "NONE" - # ---------------- Step 5: Read firmware version on DEV2 ---------------- - - name: Read firmware version from DEV2 + # ---------------- Step 5 Read firmware version on DEV2 ---------------- + - name: Read firmware version or banner from DEV2 delegate_to: localhost ansible.builtin.shell: | sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \ @@ -334,58 +334,58 @@ changed_when: false ignore_errors: true - - name: Show firmware version readout + - name: Debug show raw firmware readout from DEV2 delegate_to: localhost ansible.builtin.debug: msg: "Firmware version on DEV2 {{ (dev2_fwver.stdout | default('')) | trim }}" - # ---------------- Retry metadata + success evaluation (read OK) ---------------- - - name: Normalize retry metadata for indoor checker + # ---------------- Retry metadata and success evaluation ---------------- + - name: Normalize retry control facts for indoor checker ansible.builtin.set_fact: attempt: "{{ (attempt | default(1)) | int }}" effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" correlation_id: "{{ correlation_id | default('') }}" original_emitted_at: "{{ original_emitted_at | default('') }}" - - name: Check if firmware read succeeded read_ok + - name: Compute read_ok flag based on firmware readout delegate_to: localhost ansible.builtin.set_fact: read_ok: "{{ (dev2_fwver.rc | default(1) == 0) and ((dev2_fwver.stdout | default('') | trim) | length > 0) and (not (((dev2_fwver.stdout | default('unknown')) | lower) is search('unknown'))) }}" - # ===================== Unconditional normalization + comparison ===================== + # ===================== Normalization and comparison ===================== # Compute expected_norm from target_version_full. If a whole filename is sent, - # extract "X.Y.Z-rNNNN"; otherwise keep the original trimmed string. + # extract X.Y.Z-rNNNN otherwise keep the original trimmed string. - - name: Normalize expected target step 1 compute base + - name: Normalize expected target step one compute base string delegate_to: localhost ansible.builtin.set_fact: expected_norm_step1: "{{ (target_version_full | default('') | trim) }}" - - name: Extract version core X.Y.Z-rNNNN from target if present + - name: Extract version core X dot Y dot Z dash rNNNN from target if present delegate_to: localhost ansible.builtin.set_fact: expected_norm_core_list: "{{ (target_version_full | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}" - - name: Choose first core if present + - name: Choose first extracted core if available delegate_to: localhost ansible.builtin.set_fact: expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}" - - name: Normalize expected target step 2 choose core if present + - name: Normalize expected target step two prefer extracted core when available delegate_to: localhost ansible.builtin.set_fact: expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}" - # 1) capture raw banner line from DEV2 - - name: Capture banner line from DEV2 + # 1 capture raw banner line from DEV2 + - name: Capture firmware banner line from DEV2 delegate_to: localhost ansible.builtin.set_fact: banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}" - # 2) normalize "... rev 6801" → "...-r6801" - - name: Normalize banner line to X.Y.Z-rNNNN suffix + # 2 normalize rev suffix to dash rNNNN + - name: Normalize banner line to X dot Y dot Z dash rNNNN suffix delegate_to: localhost ansible.builtin.set_fact: banner_norm: >- @@ -397,7 +397,7 @@ ) }} - - name: Evaluate version match normalized equality or contains + - name: Evaluate version match using normalized equality or substring delegate_to: localhost ansible.builtin.set_fact: version_match: >- @@ -410,7 +410,7 @@ ) }} - - name: Debug compare snapshot expected vs actual normalized + - name: Debug snapshot of expected and actual normalized versions delegate_to: localhost ansible.builtin.debug: msg: @@ -418,9 +418,9 @@ - "banner_norm={{ banner_norm }}" - "version_match={{ version_match | default(false) }}" - # ===================== Journaling/Tagging paths ===================== + # ===================== Journaling and tagging paths ===================== # SUCCESS path - - name: Build success journal payload + - name: Build journal payload for successful after upgrade check when: (read_ok | bool) and (version_match | bool) delegate_to: localhost ansible.builtin.set_fact: @@ -454,7 +454,7 @@ register: rmq_pub_success changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool) - - name: Build payload to add indoor-update-success tag DEV1 + - name: Build payload to add tag indoor update success on DEV1 when: journal_success_payload is defined delegate_to: localhost ansible.builtin.set_fact: @@ -463,7 +463,7 @@ task_name: "tag_add" task_result: "indoor-update-success" - - name: Publish indoor-update-success tag + - name: Publish tag indoor update success when: tag_add_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -485,7 +485,7 @@ register: rmq_tag_add_success changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool) - - name: Build payload to remove indoor-restart-scheduled tag DEV1 + - name: Build payload to remove tag indoor restart scheduled on DEV1 when: journal_success_payload is defined delegate_to: localhost ansible.builtin.set_fact: @@ -494,7 +494,7 @@ task_name: "tag_remove" task_result: "indoor-restart-scheduled" - - name: Publish indoor-restart-scheduled tag removal + - name: Publish tag removal indoor restart scheduled when: tag_remove_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -517,13 +517,13 @@ changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) # --- Normalize firmware string and set custom field on success --- - - name: Capture raw firmware banner from DEV2 for normalization + - name: Capture raw firmware banner for normalization on success path when: journal_success_payload is defined delegate_to: localhost ansible.builtin.set_fact: fw_banner_raw: "{{ banner_raw }}" - - name: Normalize firmware string for indoor_fwver e g 2.2.1 rev 6801 to 2.2.1-r6801 + - name: Normalize firmware string for indoor_fwver to X dot Y dot Z dash rNNNN when: - journal_success_payload is defined - (fw_banner_raw | default('') | length) > 0 @@ -538,13 +538,13 @@ ) }} - - name: Debug normalized firmware indoor_fwver + - name: Debug show normalized indoor firmware value when: fw_norm is defined delegate_to: localhost ansible.builtin.debug: msg: "Normalized indoor_fwver={{ fw_norm }} (from {{ fw_banner_raw }})" - - name: Publish custom_field_set indoor_fwver + - name: Publish custom field indoor_fwver to control queue when: - fw_norm is defined - (fw_norm | length) > 0 @@ -574,7 +574,7 @@ changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool) # MISMATCH path - - name: Build mismatch journal payload + - name: Build journal payload for version mismatch after upgrade when: (read_ok | bool) and (not (version_match | bool)) delegate_to: localhost ansible.builtin.set_fact: @@ -586,7 +586,7 @@ Expected='{{ expected_norm }}' Got='{{ banner_raw }}' Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - name: Publish mismatch journal to control queue + - name: Publish journal for version mismatch when: journal_mismatch_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -608,12 +608,12 @@ register: rmq_j_mismatch changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) - - name: Stop host after mismatch evaluation + - name: Stop host after mismatch path is handled when: journal_mismatch_payload is defined ansible.builtin.meta: end_host - # FAILURE / RETRY PATH - - name: Build failure journal payload indoor firmware read failed + # FAILURE and RETRY path + - name: Build journal payload for indoor firmware read failure when: not (read_ok | bool) delegate_to: localhost ansible.builtin.set_fact: @@ -625,7 +625,7 @@ fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}' Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - name: Publish failure journal to control queue + - name: Publish journal for indoor firmware read failure when: journal_fail_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -646,14 +646,14 @@ register: rmq_pub_fail changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool) - - name: Compute retry parameters 10 minutes + - name: Compute retry parameters wait ten minutes before next attempt when: not (read_ok | bool) ansible.builtin.set_fact: next_attempt: "{{ (attempt | int) + 1 }}" next_delay_sec: 600 next_delay_ms: "{{ 600000 }}" - - name: Build final gave-up journal max attempts reached fwread + - name: Build final gave up journal when max attempts reached after read failure when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int)) delegate_to: localhost ansible.builtin.set_fact: @@ -665,7 +665,7 @@ Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}. Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - name: Publish final gave-up journal fwread + - name: Publish final gave up journal after read failure when: journal_gaveup_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -687,7 +687,7 @@ changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool) # Only schedule next attempt if budget left - - name: Build delayed payload for next indoor attempt 10 min + - name: Build delayed payload for next indoor attempt wait ten minutes when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int)) delegate_to: localhost ansible.builtin.set_fact: @@ -701,7 +701,7 @@ current_delay_sec: "{{ next_delay_sec | int }}" schema_version: 1 - - name: Publish delayed next indoor attempt holding plus TTL to deviceconfig + - name: Publish delayed next indoor attempt to holding with TTL routed to deviceconfig when: delayed_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -725,28 +725,28 @@ changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) post_tasks: - - name: Cleanup tunnel and temp IP + - name: Cleanup section tunnel and temporary IP block: - ansible.builtin.debug: msg: "Cleaning up tunnel and temporary IP" changed_when: false delegate_to: localhost always: - - name: Close tunnel + - name: Close SSH ControlMaster tunnel best effort delegate_to: localhost ansible.builtin.shell: | ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true changed_when: false ignore_errors: true - - name: Remove control dir + - name: Remove temporary control directory best effort delegate_to: localhost ansible.builtin.file: path: "{{ _ctrl_dir | default('/tmp/none') }}" state: absent ignore_errors: true - - name: Remove temporary IP from DEV1 + - name: Remove temporary IP from DEV1 best effort ansible.builtin.raw: > {{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} diff --git a/files/ansible-playbooks/update-indoor.yml b/files/ansible-playbooks/update-indoor.yml index 6d27c80..5759a73 100644 --- a/files/ansible-playbooks/update-indoor.yml +++ b/files/ansible-playbooks/update-indoor.yml @@ -59,6 +59,17 @@ register: dev1_host_read changed_when: false + - name: Debug incoming parameters from consumer and defaults + delegate_to: localhost + ansible.builtin.debug: + msg: + - "rebootin={{ rebootin | default('UNSET') }}" + - "image_filename={{ image_filename | default('UNSET') }}" + - "image_md5={{ image_md5 | default('UNSET') }}" + - "image_sha256={{ image_sha256 | default('UNSET') }}" + - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}" + - "inventory_hostname={{ inventory_hostname }}" + - name: Stop early if connected DEV1 hostname != inventory ansible.builtin.meta: end_host when: (dev1_host_read.stdout | trim | length > 0) and @@ -484,6 +495,22 @@ else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$')) else -1) }} + - name: Debug reboot normalization detail + delegate_to: localhost + ansible.builtin.debug: + msg: + - "_reboot_raw={{ _reboot_raw | default('UNSET') }}" + - "_reboot_is_now={{ _reboot_is_now | default('UNSET') }}" + - "_reboot_is_int={{ _reboot_is_int | default('UNSET') }}" + - "_reboot_hours={{ _reboot_hours | default('UNSET') }}" + + - name: Warn if rebootin was not provided by consumer (debug only) + when: (rebootin | default('') | string | trim) == '' + delegate_to: localhost + ansible.builtin.debug: + msg: "WARNING: rebootin is empty or missing. Consumer likely did not pass -e rebootin=." + + # Compute reboot flags and *final* seconds value with the 20s grace rule # Rule: now/empty -> 20s; integer <=0 -> 20s; integer >0 -> hours*3600 - name: Compute reboot flags requested and seconds with 20s grace