diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml b/files/ansible-playbooks/afterupgrade_indoor_check.yml index 08ab118..34db288 100644 --- a/files/ansible-playbooks/afterupgrade_indoor_check.yml +++ b/files/ansible-playbooks/afterupgrade_indoor_check.yml @@ -51,10 +51,10 @@ - "max_attempts={{ effective_max_attempts }}" - "correlation_id={{ correlation_id }}" - "original_emitted_at={{ original_emitted_at }}" - - "target_version(full)={{ target_version_full }}" + - "target_version_full={{ target_version_full }}" # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ---- - - name: Check if TCP/22 on DEV1 is reachable + - name: Check if TCP 22 on DEV1 is reachable delegate_to: localhost ansible.builtin.shell: | nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22 @@ -63,7 +63,7 @@ ignore_errors: true # ---- If TCP down: journal + schedule next try or give up ---- - - name: Build failure journal (TCP unreachable) + mark retry + - name: Build failure journal TCP unreachable and mark retry when: nc_probe.rc != 0 delegate_to: localhost ansible.builtin.set_fact: @@ -71,12 +71,12 @@ inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- - afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}): + afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}) TCP 22 unreachable. Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' _needs_retry: true - - name: Publish failure journal (TCP unreachable) + - name: Publish failure journal TCP unreachable when: nc_probe.rc != 0 delegate_to: localhost ansible.builtin.uri: @@ -96,7 +96,7 @@ register: rmq_j_tcp_fail changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) - - name: Compute next-attempt delay (10 minutes) and counters + - name: Compute next-attempt delay 10 minutes and counters when: (_needs_retry | default(false)) | bool delegate_to: localhost ansible.builtin.set_fact: @@ -104,7 +104,7 @@ next_delay_sec: 600 next_delay_ms: 600000 - - name: Build final gave-up journal (max attempts reached) + - name: Build final gave-up journal max attempts reached when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) delegate_to: localhost ansible.builtin.set_fact: @@ -112,7 +112,7 @@ inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- - afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}) Exhausted attempts. Last error path=TCP. Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' @@ -140,7 +140,7 @@ when: journal_gaveup_payload is defined ansible.builtin.meta: end_host - - name: Build delayed after-upgrade payload for next attempt (10m) + - name: Build delayed after-upgrade payload for next attempt 10m when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) delegate_to: localhost ansible.builtin.set_fact: @@ -155,7 +155,7 @@ current_delay_sec: "{{ next_delay_sec | int }}" schema_version: 1 - - name: Publish delayed next attempt to holding (dead-letters to deviceconfig) + - name: Publish delayed next attempt to holding dead-letters to deviceconfig when: delayed_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -198,7 +198,7 @@ ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) # ---------------- Step 2: Setup temporary IP for reachability ---------------- - - name: Add temporary IP on DEV1 (ignore if exists) + - name: Add temporary IP on DEV1 ignore if exists ansible.builtin.raw: > {{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} @@ -251,14 +251,14 @@ register: start_tunnel changed_when: true - # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) --- + # --- settle + check ControlMaster + TCP probe --- - name: Small delay for tunnel to settle delegate_to: localhost ansible.builtin.wait_for: timeout: 1 changed_when: false - - name: Verify tunnel master running (ssh -O check) + - name: Verify tunnel master running ssh O check delegate_to: localhost ansible.builtin.shell: | set -e @@ -337,7 +337,7 @@ - name: Show firmware version readout delegate_to: localhost ansible.builtin.debug: - msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" + msg: "Firmware version on DEV2 {{ (dev2_fwver.stdout | default('')) | trim }}" # ---------------- Retry metadata + success evaluation (read OK) ---------------- - name: Normalize retry metadata for indoor checker @@ -347,45 +347,44 @@ correlation_id: "{{ correlation_id | default('') }}" original_emitted_at: "{{ original_emitted_at | default('') }}" - - name: Check if firmware read succeeded (read_ok) + - name: Check if firmware read succeeded read_ok delegate_to: localhost ansible.builtin.set_fact: read_ok: "{{ (dev2_fwver.rc | default(1) == 0) and ((dev2_fwver.stdout | default('') | trim) | length > 0) - and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}" + and (not (((dev2_fwver.stdout | default('unknown')) | lower) is search('unknown'))) }}" - # ===================== (NEW) Unconditional normalization + comparison ===================== + # ===================== Unconditional normalization + comparison ===================== # Compute expected_norm from target_version_full. If the full filename-like string is sent, - # we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original. + # extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original. - - name: Normalize expected target string (step 1: compute components) + - name: Normalize expected target string step 1 compute components delegate_to: localhost ansible.builtin.set_fact: expected_norm_step1: "{{ (target_version_full | default('') | trim) }}" - expected_norm_core: >- - {{ - (target_version_full | default('') | - regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1')) - | default('', true) - }} + expected_norm_core_list: "{{ (target_version_full | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}" + expected_norm_core: "{{ (expected_norm_core_list | length > 0) | ternary(expected_norm_core_list[0], '') }}" - - name: Normalize expected target string (step 2: choose core if present) + - name: Normalize expected target string step 2 choose core if present delegate_to: localhost ansible.builtin.set_fact: expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}" - # Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801" - - name: Normalize banner/firmux string from DEV2 + # Normalize banner/firmux from DEV2 convert '... rev 6801' to '...-r6801' + - name: Normalize banner firmux string from DEV2 delegate_to: localhost ansible.builtin.set_fact: banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}" banner_norm: >- {{ - (banner_raw | lower is search('-r[0-9]+$')) - | ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))) + (((banner_raw | lower) is search('-r[0-9]+$'))) + | ternary( + banner_raw, + (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) + ) }} - - name: Evaluate version match (normalized equality or contains) + - name: Evaluate version match normalized equality or contains delegate_to: localhost ansible.builtin.set_fact: version_match: >- @@ -393,17 +392,17 @@ (expected_norm | length > 0) and ( (banner_norm == expected_norm) - or (banner_norm is search(expected_norm)) - or (expected_norm is search(banner_norm)) + or ((banner_norm | lower) is search((expected_norm | lower))) + or ((expected_norm | lower) is search((banner_norm | lower))) ) }} - - name: Debug compare snapshot (expected vs actual normalized) + - name: Debug compare snapshot expected vs actual normalized delegate_to: localhost ansible.builtin.debug: msg: - - "expected_norm='{{ expected_norm }}'" - - "banner_norm='{{ banner_norm }}'" + - "expected_norm={{ expected_norm }}" + - "banner_norm={{ banner_norm }}" - "version_match={{ version_match | default(false) }}" # ===================== Journaling/Tagging paths ===================== @@ -416,7 +415,7 @@ inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- - afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}) Banner='{{ banner_raw }}' Target='{{ expected_norm }}' Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} @@ -442,7 +441,7 @@ register: rmq_pub_success changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool) - - name: Build payload to add indoor-update-success tag (DEV1) + - name: Build payload to add indoor-update-success tag DEV1 when: journal_success_payload is defined delegate_to: localhost ansible.builtin.set_fact: @@ -473,7 +472,7 @@ register: rmq_tag_add_success changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool) - - name: Build payload to remove indoor-restart-scheduled tag (DEV1) + - name: Build payload to remove indoor-restart-scheduled tag DEV1 when: journal_success_payload is defined delegate_to: localhost ansible.builtin.set_fact: @@ -505,13 +504,13 @@ changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) # --- Normalize firmware string and set custom field on success --- - - name: Capture raw firmware banner from DEV2 (for normalization) + - name: Capture raw firmware banner from DEV2 for normalization when: journal_success_payload is defined delegate_to: localhost ansible.builtin.set_fact: fw_banner_raw: "{{ banner_raw }}" - - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801') + - name: Normalize firmware string for indoor_fwver e g 2.2.1 rev 6801 to 2.2.1-r6801 when: - journal_success_payload is defined - (fw_banner_raw | default('') | length) > 0 @@ -519,16 +518,18 @@ ansible.builtin.set_fact: fw_norm: >- {{ - fw_banner_raw - if (fw_banner_raw | lower is search('-r[0-9]+$')) - else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) + (fw_banner_raw | lower) is search('-r[0-9]+$') + | ternary( + fw_banner_raw, + (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) + ) }} - - name: Debug normalized firmware (indoor_fwver) + - name: Debug normalized firmware indoor_fwver when: fw_norm is defined delegate_to: localhost ansible.builtin.debug: - msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')" + msg: "Normalized indoor_fwver={{ fw_norm }} (from {{ fw_banner_raw }})" - name: Publish custom_field_set indoor_fwver when: @@ -568,7 +569,7 @@ inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- - afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}) Expected='{{ expected_norm }}' Got='{{ banner_raw }}' Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} @@ -599,7 +600,7 @@ ansible.builtin.meta: end_host # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart - - name: Build failure journal payload (indoor firmware read failed) + - name: Build failure journal payload indoor firmware read failed when: not (read_ok | bool) delegate_to: localhost ansible.builtin.set_fact: @@ -607,7 +608,7 @@ inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- - afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}): + afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}) fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}' Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} @@ -633,14 +634,14 @@ register: rmq_pub_fail changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool) - - name: Compute retry parameters (10 minutes) + - name: Compute retry parameters 10 minutes when: not (read_ok | bool) ansible.builtin.set_fact: next_attempt: "{{ (attempt | int) + 1 }}" next_delay_sec: 600 next_delay_ms: "{{ 600000 }}" - - name: Build final gave-up journal (max attempts reached) + - name: Build final gave-up journal max attempts reached fwread when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int)) delegate_to: localhost ansible.builtin.set_fact: @@ -648,11 +649,11 @@ inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- - afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}) Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}. Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - name: Publish final gave-up journal + - name: Publish final gave-up journal fwread when: journal_gaveup_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -675,7 +676,7 @@ changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool) # Only schedule next attempt if we still have budget left - - name: Build delayed payload for next indoor attempt (10 min) + - name: Build delayed payload for next indoor attempt 10 min when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int)) delegate_to: localhost ansible.builtin.set_fact: @@ -689,7 +690,7 @@ current_delay_sec: "{{ next_delay_sec | int }}" schema_version: 1 - - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig) + - name: Publish delayed next indoor attempt holding plus TTL to deviceconfig when: delayed_payload is defined delegate_to: localhost ansible.builtin.uri: @@ -717,7 +718,7 @@ - name: Cleanup tunnel and temp IP block: - ansible.builtin.debug: - msg: "Cleaning up tunnel + temp IP" + msg: "Cleaning up tunnel and temporary IP" changed_when: false delegate_to: localhost always: