pavel/ansible-worker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1818

Browse Source
This commit is contained in:
pavel
2025-10-22 18:18:22 +03:00
parent b565fa781a
commit 4a05b151b8
1 changed files with 57 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
files/ansible-playbooks/afterupgrade_indoor_check.yml
View File

@@ -51,10 +51,10 @@
- "max_attempts={{ effective_max_attempts }}"
- "correlation_id={{ correlation_id }}"
- "original_emitted_at={{ original_emitted_at }}"
- "target_version(full)={{ target_version_full }}"
- "target_version_full={{ target_version_full }}"
# ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
- name: Check if TCP/22 on DEV1 is reachable
- name: Check if TCP 22 on DEV1 is reachable
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
@@ -63,7 +63,7 @@
ignore_errors: true
# ---- If TCP down: journal + schedule next try or give up ----
- name: Build failure journal (TCP unreachable) + mark retry
- name: Build failure journal TCP unreachable and mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
@@ -71,12 +71,12 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }})
TCP 22 unreachable. Correlation={{ correlation_id }}
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (TCP unreachable)
- name: Publish failure journal TCP unreachable
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
@@ -96,7 +96,7 @@
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
- name: Compute next-attempt delay (10 minutes) and counters
- name: Compute next-attempt delay 10 minutes and counters
when: (_needs_retry | default(false)) | bool
delegate_to: localhost
ansible.builtin.set_fact:
@@ -104,7 +104,7 @@
next_delay_sec: 600
next_delay_ms: 600000
- name: Build final gave-up journal (max attempts reached)
- name: Build final gave-up journal max attempts reached
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -112,7 +112,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }})
Exhausted attempts. Last error path=TCP.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
@@ -140,7 +140,7 @@
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
- name: Build delayed after-upgrade payload for next attempt (10m)
- name: Build delayed after-upgrade payload for next attempt 10m
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -155,7 +155,7 @@
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
- name: Publish delayed next attempt to holding dead-letters to deviceconfig
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -198,7 +198,7 @@
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
# ---------------- Step 2: Setup temporary IP for reachability ----------------
- name: Add temporary IP on DEV1 (ignore if exists)
- name: Add temporary IP on DEV1 ignore if exists
ansible.builtin.raw: >
{{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
@@ -251,14 +251,14 @@
register: start_tunnel
changed_when: true
# --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
# --- settle + check ControlMaster + TCP probe ---
- name: Small delay for tunnel to settle
delegate_to: localhost
ansible.builtin.wait_for:
timeout: 1
changed_when: false
- name: Verify tunnel master running (ssh -O check)
- name: Verify tunnel master running ssh O check
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -337,7 +337,7 @@
- name: Show firmware version readout
delegate_to: localhost
ansible.builtin.debug:
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
msg: "Firmware version on DEV2 {{ (dev2_fwver.stdout | default('')) | trim }}"
# ---------------- Retry metadata + success evaluation (read OK) ----------------
- name: Normalize retry metadata for indoor checker
@@ -347,45 +347,44 @@
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
- name: Check if firmware read succeeded (read_ok)
- name: Check if firmware read succeeded read_ok
delegate_to: localhost
ansible.builtin.set_fact:
read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
and ((dev2_fwver.stdout | default('') | trim) | length > 0)
and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
and (not (((dev2_fwver.stdout | default('unknown')) | lower) is search('unknown'))) }}"
# ===================== (NEW) Unconditional normalization + comparison =====================
# ===================== Unconditional normalization + comparison =====================
# Compute expected_norm from target_version_full. If the full filename-like string is sent,
# we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
# extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
- name: Normalize expected target string (step 1: compute components)
- name: Normalize expected target string step 1 compute components
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
expected_norm_core: >-
{{
(target_version_full | default('') |
regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1'))
| default('', true)
}}
expected_norm_core_list: "{{ (target_version_full | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
expected_norm_core: "{{ (expected_norm_core_list | length > 0) | ternary(expected_norm_core_list[0], '') }}"
- name: Normalize expected target string (step 2: choose core if present)
- name: Normalize expected target string step 2 choose core if present
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
# Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801"
- name: Normalize banner/firmux string from DEV2
# Normalize banner/firmux from DEV2 convert '... rev 6801' to '...-r6801'
- name: Normalize banner firmux string from DEV2
delegate_to: localhost
ansible.builtin.set_fact:
banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
banner_norm: >-
{{
(banner_raw | lower is search('-r[0-9]+$'))
| ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')))
(((banner_raw | lower) is search('-r[0-9]+$')))
| ternary(
banner_raw,
(banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
)
}}
- name: Evaluate version match (normalized equality or contains)
- name: Evaluate version match normalized equality or contains
delegate_to: localhost
ansible.builtin.set_fact:
version_match: >-
@@ -393,17 +392,17 @@
(expected_norm | length > 0)
and (
(banner_norm == expected_norm)
or (banner_norm is search(expected_norm))
or (expected_norm is search(banner_norm))
or ((banner_norm | lower) is search((expected_norm | lower)))
or ((expected_norm | lower) is search((banner_norm | lower)))
)
}}
- name: Debug compare snapshot (expected vs actual normalized)
- name: Debug compare snapshot expected vs actual normalized
delegate_to: localhost
ansible.builtin.debug:
msg:
- "expected_norm='{{ expected_norm }}'"
- "banner_norm='{{ banner_norm }}'"
- "expected_norm={{ expected_norm }}"
- "banner_norm={{ banner_norm }}"
- "version_match={{ version_match | default(false) }}"
# ===================== Journaling/Tagging paths =====================
@@ -416,7 +415,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }})
Banner='{{ banner_raw }}' Target='{{ expected_norm }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
@@ -442,7 +441,7 @@
register: rmq_pub_success
changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
- name: Build payload to add indoor-update-success tag (DEV1)
- name: Build payload to add indoor-update-success tag DEV1
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
@@ -473,7 +472,7 @@
register: rmq_tag_add_success
changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
- name: Build payload to remove indoor-restart-scheduled tag (DEV1)
- name: Build payload to remove indoor-restart-scheduled tag DEV1
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
@@ -505,13 +504,13 @@
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
# --- Normalize firmware string and set custom field on success ---
- name: Capture raw firmware banner from DEV2 (for normalization)
- name: Capture raw firmware banner from DEV2 for normalization
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
fw_banner_raw: "{{ banner_raw }}"
- name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
- name: Normalize firmware string for indoor_fwver e g 2.2.1 rev 6801 to 2.2.1-r6801
when:
- journal_success_payload is defined
- (fw_banner_raw | default('') | length) > 0
@@ -519,16 +518,18 @@
ansible.builtin.set_fact:
fw_norm: >-
{{
fw_banner_raw
if (fw_banner_raw | lower is search('-r[0-9]+$'))
else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
(fw_banner_raw | lower) is search('-r[0-9]+$')
| ternary(
fw_banner_raw,
(fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
)
}}
- name: Debug normalized firmware (indoor_fwver)
- name: Debug normalized firmware indoor_fwver
when: fw_norm is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
msg: "Normalized indoor_fwver={{ fw_norm }} (from {{ fw_banner_raw }})"
- name: Publish custom_field_set indoor_fwver
when:
@@ -568,7 +569,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }})
Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
@@ -599,7 +600,7 @@
ansible.builtin.meta: end_host
# FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
- name: Build failure journal payload (indoor firmware read failed)
- name: Build failure journal payload indoor firmware read failed
when: not (read_ok | bool)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -607,7 +608,7 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }})
fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
@@ -633,14 +634,14 @@
register: rmq_pub_fail
changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
- name: Compute retry parameters (10 minutes)
- name: Compute retry parameters 10 minutes
when: not (read_ok | bool)
ansible.builtin.set_fact:
next_attempt: "{{ (attempt | int) + 1 }}"
next_delay_sec: 600
next_delay_ms: "{{ 600000 }}"
- name: Build final gave-up journal (max attempts reached)
- name: Build final gave-up journal max attempts reached fwread
when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
delegate_to: localhost
ansible.builtin.set_fact:
@@ -648,11 +649,11 @@
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }})
Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish final gave-up journal
- name: Publish final gave-up journal fwread
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -675,7 +676,7 @@
changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
# Only schedule next attempt if we still have budget left
- name: Build delayed payload for next indoor attempt (10 min)
- name: Build delayed payload for next indoor attempt 10 min
when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
delegate_to: localhost
ansible.builtin.set_fact:
@@ -689,7 +690,7 @@
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
- name: Publish delayed next indoor attempt holding plus TTL to deviceconfig
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -717,7 +718,7 @@
- name: Cleanup tunnel and temp IP
block:
- ansible.builtin.debug:
msg: "Cleaning up tunnel + temp IP"
msg: "Cleaning up tunnel and temporary IP"
changed_when: false
delegate_to: localhost
always: