pavel/ansible-worker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1818

Browse Source
This commit is contained in:
pavel
2025-10-22 18:18:22 +03:00
parent b565fa781a
commit 4a05b151b8
1 changed files with 57 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
files/ansible-playbooks/afterupgrade_indoor_check.yml
View File

@@ -51,10 +51,10 @@
- "max_attempts={{ effective_max_attempts }}" - "max_attempts={{ effective_max_attempts }}"
- "correlation_id={{ correlation_id }}" - "correlation_id={{ correlation_id }}"
- "original_emitted_at={{ original_emitted_at }}" - "original_emitted_at={{ original_emitted_at }}"
- "target_version(full)={{ target_version_full }}" - "target_version_full={{ target_version_full }}"
# ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ---- # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
- name: Check if TCP/22 on DEV1 is reachable - name: Check if TCP 22 on DEV1 is reachable
delegate_to: localhost delegate_to: localhost
ansible.builtin.shell: | ansible.builtin.shell: |
nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22 nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
@@ -63,7 +63,7 @@
ignore_errors: true ignore_errors: true
# ---- If TCP down: journal + schedule next try or give up ---- # ---- If TCP down: journal + schedule next try or give up ----
- name: Build failure journal (TCP unreachable) + mark retry - name: Build failure journal TCP unreachable and mark retry
when: nc_probe.rc != 0 when: nc_probe.rc != 0
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -71,12 +71,12 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add" task_name: "journal_add"
task_result: >- task_result: >-
afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}): afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }})
TCP 22 unreachable. Correlation={{ correlation_id }} TCP 22 unreachable. Correlation={{ correlation_id }}
Original={{ original_emitted_at }} Target='{{ target_version_full }}' Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true _needs_retry: true
- name: Publish failure journal (TCP unreachable) - name: Publish failure journal TCP unreachable
when: nc_probe.rc != 0 when: nc_probe.rc != 0
delegate_to: localhost delegate_to: localhost
ansible.builtin.uri: ansible.builtin.uri:
@@ -96,7 +96,7 @@
register: rmq_j_tcp_fail register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
- name: Compute next-attempt delay (10 minutes) and counters - name: Compute next-attempt delay 10 minutes and counters
when: (_needs_retry | default(false)) | bool when: (_needs_retry | default(false)) | bool
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -104,7 +104,7 @@
next_delay_sec: 600 next_delay_sec: 600
next_delay_ms: 600000 next_delay_ms: 600000
- name: Build final gave-up journal (max attempts reached) - name: Build final gave-up journal max attempts reached
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -112,7 +112,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add" task_name: "journal_add"
task_result: >- task_result: >-
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }})
Exhausted attempts. Last error path=TCP. Exhausted attempts. Last error path=TCP.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
@@ -140,7 +140,7 @@
when: journal_gaveup_payload is defined when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host ansible.builtin.meta: end_host
- name: Build delayed after-upgrade payload for next attempt (10m) - name: Build delayed after-upgrade payload for next attempt 10m
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -155,7 +155,7 @@
current_delay_sec: "{{ next_delay_sec | int }}" current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1 schema_version: 1
- name: Publish delayed next attempt to holding (dead-letters to deviceconfig) - name: Publish delayed next attempt to holding dead-letters to deviceconfig
when: delayed_payload is defined when: delayed_payload is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.uri: ansible.builtin.uri:
@@ -198,7 +198,7 @@
((dev1_host_read.stdout | trim) != (inventory_hostname | string)) ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
# ---------------- Step 2: Setup temporary IP for reachability ---------------- # ---------------- Step 2: Setup temporary IP for reachability ----------------
- name: Add temporary IP on DEV1 (ignore if exists) - name: Add temporary IP on DEV1 ignore if exists
ansible.builtin.raw: > ansible.builtin.raw: >
{{ pathprefix }} {{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
@@ -251,14 +251,14 @@
register: start_tunnel register: start_tunnel
changed_when: true changed_when: true
# --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) --- # --- settle + check ControlMaster + TCP probe ---
- name: Small delay for tunnel to settle - name: Small delay for tunnel to settle
delegate_to: localhost delegate_to: localhost
ansible.builtin.wait_for: ansible.builtin.wait_for:
timeout: 1 timeout: 1
changed_when: false changed_when: false
- name: Verify tunnel master running (ssh -O check) - name: Verify tunnel master running ssh O check
delegate_to: localhost delegate_to: localhost
ansible.builtin.shell: | ansible.builtin.shell: |
set -e set -e
@@ -337,7 +337,7 @@
- name: Show firmware version readout - name: Show firmware version readout
delegate_to: localhost delegate_to: localhost
ansible.builtin.debug: ansible.builtin.debug:
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" msg: "Firmware version on DEV2 {{ (dev2_fwver.stdout | default('')) | trim }}"
# ---------------- Retry metadata + success evaluation (read OK) ---------------- # ---------------- Retry metadata + success evaluation (read OK) ----------------
- name: Normalize retry metadata for indoor checker - name: Normalize retry metadata for indoor checker
@@ -347,45 +347,44 @@
correlation_id: "{{ correlation_id | default('') }}" correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}" original_emitted_at: "{{ original_emitted_at | default('') }}"
- name: Check if firmware read succeeded (read_ok) - name: Check if firmware read succeeded read_ok
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
read_ok: "{{ (dev2_fwver.rc | default(1) == 0) read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
and ((dev2_fwver.stdout | default('') | trim) | length > 0) and ((dev2_fwver.stdout | default('') | trim) | length > 0)
and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}" and (not (((dev2_fwver.stdout | default('unknown')) | lower) is search('unknown'))) }}"
# ===================== (NEW) Unconditional normalization + comparison ===================== # ===================== Unconditional normalization + comparison =====================
# Compute expected_norm from target_version_full. If the full filename-like string is sent, # Compute expected_norm from target_version_full. If the full filename-like string is sent,
# we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original. # extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
- name: Normalize expected target string (step 1: compute components) - name: Normalize expected target string step 1 compute components
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
expected_norm_step1: "{{ (target_version_full | default('') | trim) }}" expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
expected_norm_core: >- expected_norm_core_list: "{{ (target_version_full | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
{{ expected_norm_core: "{{ (expected_norm_core_list | length > 0) | ternary(expected_norm_core_list[0], '') }}"
(target_version_full | default('') |
regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1'))
| default('', true)
}}
- name: Normalize expected target string (step 2: choose core if present) - name: Normalize expected target string step 2 choose core if present
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}" expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
# Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801" # Normalize banner/firmux from DEV2 convert '... rev 6801' to '...-r6801'
- name: Normalize banner/firmux string from DEV2 - name: Normalize banner firmux string from DEV2
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}" banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
banner_norm: >- banner_norm: >-
{{ {{
(banner_raw | lower is search('-r[0-9]+$')) (((banner_raw | lower) is search('-r[0-9]+$')))
| ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))) | ternary(
banner_raw,
(banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
)
}} }}
- name: Evaluate version match (normalized equality or contains) - name: Evaluate version match normalized equality or contains
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
version_match: >- version_match: >-
@@ -393,17 +392,17 @@
(expected_norm | length > 0) (expected_norm | length > 0)
and ( and (
(banner_norm == expected_norm) (banner_norm == expected_norm)
or (banner_norm is search(expected_norm)) or ((banner_norm | lower) is search((expected_norm | lower)))
or (expected_norm is search(banner_norm)) or ((expected_norm | lower) is search((banner_norm | lower)))
) )
}} }}
- name: Debug compare snapshot (expected vs actual normalized) - name: Debug compare snapshot expected vs actual normalized
delegate_to: localhost delegate_to: localhost
ansible.builtin.debug: ansible.builtin.debug:
msg: msg:
- "expected_norm='{{ expected_norm }}'" - "expected_norm={{ expected_norm }}"
- "banner_norm='{{ banner_norm }}'" - "banner_norm={{ banner_norm }}"
- "version_match={{ version_match | default(false) }}" - "version_match={{ version_match | default(false) }}"
# ===================== Journaling/Tagging paths ===================== # ===================== Journaling/Tagging paths =====================
@@ -416,7 +415,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add" task_name: "journal_add"
task_result: >- task_result: >-
afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }})
Banner='{{ banner_raw }}' Target='{{ expected_norm }}' Banner='{{ banner_raw }}' Target='{{ expected_norm }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
@@ -442,7 +441,7 @@
register: rmq_pub_success register: rmq_pub_success
changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool) changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
- name: Build payload to add indoor-update-success tag (DEV1) - name: Build payload to add indoor-update-success tag DEV1
when: journal_success_payload is defined when: journal_success_payload is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -473,7 +472,7 @@
register: rmq_tag_add_success register: rmq_tag_add_success
changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool) changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
- name: Build payload to remove indoor-restart-scheduled tag (DEV1) - name: Build payload to remove indoor-restart-scheduled tag DEV1
when: journal_success_payload is defined when: journal_success_payload is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -505,13 +504,13 @@
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
# --- Normalize firmware string and set custom field on success --- # --- Normalize firmware string and set custom field on success ---
- name: Capture raw firmware banner from DEV2 (for normalization) - name: Capture raw firmware banner from DEV2 for normalization
when: journal_success_payload is defined when: journal_success_payload is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
fw_banner_raw: "{{ banner_raw }}" fw_banner_raw: "{{ banner_raw }}"
- name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801') - name: Normalize firmware string for indoor_fwver e g 2.2.1 rev 6801 to 2.2.1-r6801
when: when:
- journal_success_payload is defined - journal_success_payload is defined
- (fw_banner_raw | default('') | length) > 0 - (fw_banner_raw | default('') | length) > 0
@@ -519,16 +518,18 @@
ansible.builtin.set_fact: ansible.builtin.set_fact:
fw_norm: >- fw_norm: >-
{{ {{
fw_banner_raw (fw_banner_raw | lower) is search('-r[0-9]+$')
if (fw_banner_raw | lower is search('-r[0-9]+$')) | ternary(
else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) fw_banner_raw,
(fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
)
}} }}
- name: Debug normalized firmware (indoor_fwver) - name: Debug normalized firmware indoor_fwver
when: fw_norm is defined when: fw_norm is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.debug: ansible.builtin.debug:
msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')" msg: "Normalized indoor_fwver={{ fw_norm }} (from {{ fw_banner_raw }})"
- name: Publish custom_field_set indoor_fwver - name: Publish custom_field_set indoor_fwver
when: when:
@@ -568,7 +569,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add" task_name: "journal_add"
task_result: >- task_result: >-
afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }})
Expected='{{ expected_norm }}' Got='{{ banner_raw }}' Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
@@ -599,7 +600,7 @@
ansible.builtin.meta: end_host ansible.builtin.meta: end_host
# FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
- name: Build failure journal payload (indoor firmware read failed) - name: Build failure journal payload indoor firmware read failed
when: not (read_ok | bool) when: not (read_ok | bool)
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -607,7 +608,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add" task_name: "journal_add"
task_result: >- task_result: >-
afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}): afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }})
fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}' fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
@@ -633,14 +634,14 @@
register: rmq_pub_fail register: rmq_pub_fail
changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool) changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
- name: Compute retry parameters (10 minutes) - name: Compute retry parameters 10 minutes
when: not (read_ok | bool) when: not (read_ok | bool)
ansible.builtin.set_fact: ansible.builtin.set_fact:
next_attempt: "{{ (attempt | int) + 1 }}" next_attempt: "{{ (attempt | int) + 1 }}"
next_delay_sec: 600 next_delay_sec: 600
next_delay_ms: "{{ 600000 }}" next_delay_ms: "{{ 600000 }}"
- name: Build final gave-up journal (max attempts reached) - name: Build final gave-up journal max attempts reached fwread
when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int)) when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -648,11 +649,11 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add" task_name: "journal_add"
task_result: >- task_result: >-
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }})
Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}. Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish final gave-up journal - name: Publish final gave-up journal fwread
when: journal_gaveup_payload is defined when: journal_gaveup_payload is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.uri: ansible.builtin.uri:
@@ -675,7 +676,7 @@
changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool) changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
# Only schedule next attempt if we still have budget left # Only schedule next attempt if we still have budget left
- name: Build delayed payload for next indoor attempt (10 min) - name: Build delayed payload for next indoor attempt 10 min
when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int)) when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
delegate_to: localhost delegate_to: localhost
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -689,7 +690,7 @@
current_delay_sec: "{{ next_delay_sec | int }}" current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1 schema_version: 1
- name: Publish delayed next indoor attempt (holding + TTL → deviceconfig) - name: Publish delayed next indoor attempt holding plus TTL to deviceconfig
when: delayed_payload is defined when: delayed_payload is defined
delegate_to: localhost delegate_to: localhost
ansible.builtin.uri: ansible.builtin.uri:
@@ -717,7 +718,7 @@
- name: Cleanup tunnel and temp IP - name: Cleanup tunnel and temp IP
block: block:
- ansible.builtin.debug: - ansible.builtin.debug:
msg: "Cleaning up tunnel + temp IP" msg: "Cleaning up tunnel and temporary IP"
changed_when: false changed_when: false
delegate_to: localhost delegate_to: localhost
always: always: