pavel/ansible-worker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2135

Browse Source
This commit is contained in:
pavel
2025-10-22 21:35:07 +03:00
parent 5743943ce5
commit e1653cf1b2
2 changed files with 109 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
files/ansible-playbooks/afterupgrade_indoor_check.yml
View File

@@ -1,5 +1,5 @@
---
- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
- name: After upgrade verification for indoor DEV2 via DEV1 tunnel
hosts: all
gather_facts: no
@@ -31,8 +31,8 @@
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
tasks:
# ---- Normalize metadata (from delayed message) ----
- name: Normalize after-upgrade metadata
# ---- Normalize metadata from delayed message ----
- name: Normalize after upgrade metadata into facts
delegate_to: localhost
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
@@ -42,8 +42,8 @@
target_version: "{{ target_version | default('') }}"
target_version_full: "{{ target_version | default('') }}"
# (NEW) Show what we received from the scheduler
- name: Debug received scheduler metadata
# Visibility of what scheduler sent
- name: Debug show received scheduler metadata
delegate_to: localhost
ansible.builtin.debug:
msg:
@@ -53,8 +53,8 @@
- "original_emitted_at={{ original_emitted_at }}"
- "target_version_full={{ target_version_full }}"
# ---- Controller-side TCP probe to DEV1 ----
- name: Check if TCP 22 on DEV1 is reachable
# ---- Controller side TCP probe to DEV1 ----
- name: Probe DEV1 TCP 22 from controller
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
@@ -62,8 +62,8 @@
changed_when: false
ignore_errors: true
# ---- If TCP down: journal + schedule next try or give up ----
- name: Build failure journal TCP unreachable and mark retry
# ---- If TCP down publish journal and schedule next try or give up ----
- name: Build journal payload for DEV1 TCP unreachable and mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
@@ -76,7 +76,7 @@
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal TCP unreachable
- name: Publish journal for DEV1 TCP unreachable
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
@@ -96,7 +96,7 @@
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
- name: Compute next-attempt delay 10 minutes and counters
- name: Compute next attempt delay ten minutes and counters
when: (_needs_retry | default(false)) | bool
delegate_to: localhost
ansible.builtin.set_fact:
@@ -104,7 +104,7 @@
next_delay_sec: 600
next_delay_ms: 600000
- name: Build final gave-up journal max attempts reached
- name: Build final gave up journal when max attempts reached
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -116,7 +116,7 @@
Exhausted attempts. Last error path=TCP.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
- name: Publish final gave-up journal
- name: Publish final gave up journal after TCP failures
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -136,11 +136,11 @@
register: rmq_j_gaveup
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
- name: Stop host after final gave-up
- name: Stop host after final gave up due to TCP failure
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
- name: Build delayed after-upgrade payload for next attempt 10m
- name: Build delayed payload for next attempt after upgrade ten minutes
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -155,7 +155,7 @@
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding dead-letters to deviceconfig
- name: Publish delayed next attempt to holding exchange with TTL to deviceconfig
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -183,8 +183,8 @@
ansible.builtin.meta: end_host
# ---------------- Step 1: Read DEV1 hostname and sanity ----------------
- name: Read DEV1 hostname
# ---------------- Step 1 Read DEV1 hostname and sanity ----------------
- name: Read DEV1 hostname from kernel procfs
when: nc_probe.rc == 0
ansible.builtin.raw: >
{{ pathprefix }}
@@ -192,13 +192,13 @@
register: dev1_host_read
changed_when: false
- name: Stop if DEV1 hostname mismatch
- name: Stop if DEV1 hostname does not match inventory hostname
ansible.builtin.meta: end_host
when: (dev1_host_read.stdout | trim | length > 0) and
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
# ---------------- Step 2: Setup temporary IP for reachability ----------------
- name: Add temporary IP on DEV1 ignore if exists
# ---------------- Step 2 Setup temporary IP for reachability ----------------
- name: Add temporary IP on DEV1 ignore if address already exists
ansible.builtin.raw: >
{{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
@@ -209,8 +209,8 @@
('File exists' not in (add_ip.stdout | default(''))) and
('File exists' not in (add_ip.stderr | default('')))
# ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
- name: Pick random free local port
# ---------------- Step 3 Start SSH tunnel via DEV1 ----------------
- name: Pick random free local port for SSH tunnel
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -222,22 +222,22 @@
register: pick_port
changed_when: false
- name: Stop if no free local port
- name: Stop if no free local port was found
ansible.builtin.meta: end_host
when: (pick_port.stdout | trim | length) == 0
- name: Record chosen local port and control dir
- name: Record chosen local port and create control directory
delegate_to: localhost
ansible.builtin.set_fact:
_local_port: "{{ pick_port.stdout | trim }}"
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
- name: Build tunnel control socket path
- name: Build SSH control socket path for tunnel
delegate_to: localhost
ansible.builtin.set_fact:
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
- name: Start SSH ControlMaster tunnel via DEV1
- name: Start SSH ControlMaster tunnel via DEV1 to DEV2
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -251,14 +251,14 @@
register: start_tunnel
changed_when: true
# --- settle + check ControlMaster + TCP probe ---
- name: Small delay for tunnel to settle
# --- settle and check ControlMaster and TCP probe ---
- name: Pause briefly to let SSH tunnel settle
delegate_to: localhost
ansible.builtin.wait_for:
timeout: 1
changed_when: false
- name: Verify tunnel master running ssh O check
- name: Verify tunnel ControlMaster is running with ssh O check
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -267,14 +267,14 @@
register: tun_check
changed_when: false
- name: Debug tunnel check
- name: Debug show tunnel master check result
delegate_to: localhost
ansible.builtin.debug:
msg:
- "tunnel_check.rc={{ tun_check.rc }}"
- "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
- name: Sanity confirm tunnel TCP reachability to DEV2
- name: Probe TCP reachability through tunnel to DEV2
delegate_to: localhost
ansible.builtin.shell: |
set -e
@@ -283,7 +283,7 @@
changed_when: false
ignore_errors: true
- name: Debug reachability result
- name: Debug show reachability probe result through tunnel
delegate_to: localhost
ansible.builtin.debug:
msg:
@@ -291,12 +291,12 @@
- "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
- "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
- name: Stop if tunnel TCP check failed
- name: Stop if tunnel reachability check failed
ansible.builtin.meta: end_host
when: nc_probe.rc != 0
# ---------------- Step 4: Determine working password for DEV2 ----------------
- name: Try both passfiles for DEV2
# ---------------- Step 4 Determine working password for DEV2 ----------------
- name: Try provided passfiles to authenticate to DEV2
delegate_to: localhost
ansible.builtin.shell: |
for f in {{ dev2_passfiles | join(' ') }}; do
@@ -308,23 +308,23 @@
register: dev2_passfile_try
changed_when: false
- name: Save selected DEV2 passfile
- name: Save selected DEV2 passfile for subsequent commands
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
changed_when: false
- name: Debug selected DEV2 passfile
- name: Debug show which DEV2 passfile was selected
delegate_to: localhost
ansible.builtin.debug:
msg: "dev2_passfile_used={{ dev2_passfile_used }}"
- name: Stop if no valid passfile found
- name: Stop if no valid DEV2 passfile was found
ansible.builtin.meta: end_host
when: dev2_passfile_used == "NONE"
# ---------------- Step 5: Read firmware version on DEV2 ----------------
- name: Read firmware version from DEV2
# ---------------- Step 5 Read firmware version on DEV2 ----------------
- name: Read firmware version or banner from DEV2
delegate_to: localhost
ansible.builtin.shell: |
sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
@@ -334,58 +334,58 @@
changed_when: false
ignore_errors: true
- name: Show firmware version readout
- name: Debug show raw firmware readout from DEV2
delegate_to: localhost
ansible.builtin.debug:
msg: "Firmware version on DEV2 {{ (dev2_fwver.stdout | default('')) | trim }}"
# ---------------- Retry metadata + success evaluation (read OK) ----------------
- name: Normalize retry metadata for indoor checker
# ---------------- Retry metadata and success evaluation ----------------
- name: Normalize retry control facts for indoor checker
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
- name: Check if firmware read succeeded read_ok
- name: Compute read_ok flag based on firmware readout
delegate_to: localhost
ansible.builtin.set_fact:
read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
and ((dev2_fwver.stdout | default('') | trim) | length > 0)
and (not (((dev2_fwver.stdout | default('unknown')) | lower) is search('unknown'))) }}"
# ===================== Unconditional normalization + comparison =====================
# ===================== Normalization and comparison =====================
# Compute expected_norm from target_version_full. If a whole filename is sent,
# extract "X.Y.Z-rNNNN"; otherwise keep the original trimmed string.
# extract X.Y.Z-rNNNN otherwise keep the original trimmed string.
- name: Normalize expected target step 1 compute base
- name: Normalize expected target step one compute base string
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
- name: Extract version core X.Y.Z-rNNNN from target if present
- name: Extract version core X dot Y dot Z dash rNNNN from target if present
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_core_list: "{{ (target_version_full | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
- name: Choose first core if present
- name: Choose first extracted core if available
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
- name: Normalize expected target step 2 choose core if present
- name: Normalize expected target step two prefer extracted core when available
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
# 1) capture raw banner line from DEV2
- name: Capture banner line from DEV2
# 1 capture raw banner line from DEV2
- name: Capture firmware banner line from DEV2
delegate_to: localhost
ansible.builtin.set_fact:
banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
# 2) normalize "... rev 6801" → "...-r6801"
- name: Normalize banner line to X.Y.Z-rNNNN suffix
# 2 normalize rev suffix to dash rNNNN
- name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
delegate_to: localhost
ansible.builtin.set_fact:
banner_norm: >-
@@ -397,7 +397,7 @@
)
}}
- name: Evaluate version match normalized equality or contains
- name: Evaluate version match using normalized equality or substring
delegate_to: localhost
ansible.builtin.set_fact:
version_match: >-
@@ -410,7 +410,7 @@
)
}}
- name: Debug compare snapshot expected vs actual normalized
- name: Debug snapshot of expected and actual normalized versions
delegate_to: localhost
ansible.builtin.debug:
msg:
@@ -418,9 +418,9 @@
- "banner_norm={{ banner_norm }}"
- "version_match={{ version_match | default(false) }}"
# ===================== Journaling/Tagging paths =====================
# ===================== Journaling and tagging paths =====================
# SUCCESS path
- name: Build success journal payload
- name: Build journal payload for successful after upgrade check
when: (read_ok | bool) and (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -454,7 +454,7 @@
register: rmq_pub_success
changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
- name: Build payload to add indoor-update-success tag DEV1
- name: Build payload to add tag indoor update success on DEV1
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
@@ -463,7 +463,7 @@
task_name: "tag_add"
task_result: "indoor-update-success"
- name: Publish indoor-update-success tag
- name: Publish tag indoor update success
when: tag_add_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -485,7 +485,7 @@
register: rmq_tag_add_success
changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
- name: Build payload to remove indoor-restart-scheduled tag DEV1
- name: Build payload to remove tag indoor restart scheduled on DEV1
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
@@ -494,7 +494,7 @@
task_name: "tag_remove"
task_result: "indoor-restart-scheduled"
- name: Publish indoor-restart-scheduled tag removal
- name: Publish tag removal indoor restart scheduled
when: tag_remove_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -517,13 +517,13 @@
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
# --- Normalize firmware string and set custom field on success ---
- name: Capture raw firmware banner from DEV2 for normalization
- name: Capture raw firmware banner for normalization on success path
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
fw_banner_raw: "{{ banner_raw }}"
- name: Normalize firmware string for indoor_fwver e g 2.2.1 rev 6801 to 2.2.1-r6801
- name: Normalize firmware string for indoor_fwver to X dot Y dot Z dash rNNNN
when:
- journal_success_payload is defined
- (fw_banner_raw | default('') | length) > 0
@@ -538,13 +538,13 @@
)
}}
- name: Debug normalized firmware indoor_fwver
- name: Debug show normalized indoor firmware value
when: fw_norm is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "Normalized indoor_fwver={{ fw_norm }} (from {{ fw_banner_raw }})"
- name: Publish custom_field_set indoor_fwver
- name: Publish custom field indoor_fwver to control queue
when:
- fw_norm is defined
- (fw_norm | length) > 0
@@ -574,7 +574,7 @@
changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
# MISMATCH path
- name: Build mismatch journal payload
- name: Build journal payload for version mismatch after upgrade
when: (read_ok | bool) and (not (version_match | bool))
delegate_to: localhost
ansible.builtin.set_fact:
@@ -586,7 +586,7 @@
Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish mismatch journal to control queue
- name: Publish journal for version mismatch
when: journal_mismatch_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -608,12 +608,12 @@
register: rmq_j_mismatch
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
- name: Stop host after mismatch evaluation
- name: Stop host after mismatch path is handled
when: journal_mismatch_payload is defined
ansible.builtin.meta: end_host
# FAILURE / RETRY PATH
- name: Build failure journal payload indoor firmware read failed
# FAILURE and RETRY path
- name: Build journal payload for indoor firmware read failure
when: not (read_ok | bool)
delegate_to: localhost
ansible.builtin.set_fact:
@@ -625,7 +625,7 @@
fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish failure journal to control queue
- name: Publish journal for indoor firmware read failure
when: journal_fail_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -646,14 +646,14 @@
register: rmq_pub_fail
changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
- name: Compute retry parameters 10 minutes
- name: Compute retry parameters wait ten minutes before next attempt
when: not (read_ok | bool)
ansible.builtin.set_fact:
next_attempt: "{{ (attempt | int) + 1 }}"
next_delay_sec: 600
next_delay_ms: "{{ 600000 }}"
- name: Build final gave-up journal max attempts reached fwread
- name: Build final gave up journal when max attempts reached after read failure
when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
delegate_to: localhost
ansible.builtin.set_fact:
@@ -665,7 +665,7 @@
Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish final gave-up journal fwread
- name: Publish final gave up journal after read failure
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -687,7 +687,7 @@
changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
# Only schedule next attempt if budget left
- name: Build delayed payload for next indoor attempt 10 min
- name: Build delayed payload for next indoor attempt wait ten minutes
when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
delegate_to: localhost
ansible.builtin.set_fact:
@@ -701,7 +701,7 @@
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next indoor attempt holding plus TTL to deviceconfig
- name: Publish delayed next indoor attempt to holding with TTL routed to deviceconfig
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
@@ -725,28 +725,28 @@
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
post_tasks:
- name: Cleanup tunnel and temp IP
- name: Cleanup section tunnel and temporary IP
block:
- ansible.builtin.debug:
msg: "Cleaning up tunnel and temporary IP"
changed_when: false
delegate_to: localhost
always:
- name: Close tunnel
- name: Close SSH ControlMaster tunnel best effort
delegate_to: localhost
ansible.builtin.shell: |
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
changed_when: false
ignore_errors: true
- name: Remove control dir
- name: Remove temporary control directory best effort
delegate_to: localhost
ansible.builtin.file:
path: "{{ _ctrl_dir | default('/tmp/none') }}"
state: absent
ignore_errors: true
- name: Remove temporary IP from DEV1
- name: Remove temporary IP from DEV1 best effort
ansible.builtin.raw: >
{{ pathprefix }}
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}