745 lines
30 KiB
YAML
745 lines
30 KiB
YAML
---
|
|
- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
|
|
hosts: all
|
|
gather_facts: no
|
|
|
|
vars:
|
|
# ---------------- BusyBox-safe path prefix ----------------
|
|
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
|
|
|
# ---------------- DEV1 (outer device) ----------------
|
|
dev1_user: "root"
|
|
dev1_pass: "wavewave"
|
|
dev1_iface: "br-wan"
|
|
|
|
# ---------------- DEV2 (indoor behind DEV1) ----------------
|
|
dev2_host: "192.168.1.1"
|
|
dev2_port: 22
|
|
dev2_side_ip: "192.168.1.11/24"
|
|
dev2_ssh_user: "root"
|
|
dev2_passfiles:
|
|
- "basicpass"
|
|
- "basicpass2"
|
|
|
|
# ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
|
|
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
|
|
|
tasks:
|
|
# ---- Normalize metadata (from delayed message) ----
|
|
- name: Normalize after-upgrade metadata
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
attempt: "{{ (attempt | default(1)) | int }}"
|
|
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
|
|
correlation_id: "{{ correlation_id | default('') }}"
|
|
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
target_version: "{{ target_version | default('') }}"
|
|
target_version_full: "{{ target_version | default('') }}"
|
|
|
|
# (NEW) Show what we received from the scheduler (for easy troubleshooting)
|
|
- name: Debug received scheduler metadata
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "attempt={{ attempt }}"
|
|
- "max_attempts={{ effective_max_attempts }}"
|
|
- "correlation_id={{ correlation_id }}"
|
|
- "original_emitted_at={{ original_emitted_at }}"
|
|
- "target_version(full)={{ target_version_full }}"
|
|
|
|
# ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
|
|
- name: Check if TCP/22 on DEV1 is reachable
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
|
|
register: nc_probe
|
|
changed_when: false
|
|
ignore_errors: true
|
|
|
|
# ---- If TCP down: journal + schedule next try or give up ----
|
|
- name: Build failure journal (TCP unreachable) + mark retry
|
|
when: nc_probe.rc != 0
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
fail_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
TCP 22 unreachable. Correlation={{ correlation_id }}
|
|
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
_needs_retry: true
|
|
|
|
- name: Publish failure journal (TCP unreachable)
|
|
when: nc_probe.rc != 0
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ fail_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_j_tcp_fail
|
|
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
|
|
|
|
- name: Compute next-attempt delay (10 minutes) and counters
|
|
when: (_needs_retry | default(false)) | bool
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
next_attempt: "{{ attempt | int + 1 }}"
|
|
next_delay_sec: 600
|
|
next_delay_ms: 600000
|
|
|
|
- name: Build final gave-up journal (max attempts reached)
|
|
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
journal_gaveup_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
Exhausted attempts. Last error path=TCP.
|
|
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
|
|
- name: Publish final gave-up journal
|
|
when: journal_gaveup_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_j_gaveup
|
|
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
|
|
|
- name: Stop host after final gave-up
|
|
when: journal_gaveup_payload is defined
|
|
ansible.builtin.meta: end_host
|
|
|
|
- name: Build delayed after-upgrade payload for next attempt (10m)
|
|
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
delayed_payload:
|
|
task_name: "afterupgrade_indoor_check"
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
attempt: "{{ next_attempt | int }}"
|
|
max_attempts: "{{ effective_max_attempts | int }}"
|
|
correlation_id: "{{ correlation_id }}"
|
|
original_emitted_at: "{{ original_emitted_at }}"
|
|
target_version: "{{ target_version_full }}"
|
|
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
schema_version: 1
|
|
|
|
- name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
|
|
when: delayed_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
correlation_id: "{{ correlation_id }}"
|
|
routing_key: "deviceconfig"
|
|
payload: "{{ delayed_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_pub_next
|
|
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
|
|
- name: Stop host after scheduling next attempt
|
|
when: (_needs_retry | default(false)) | bool
|
|
ansible.builtin.meta: end_host
|
|
|
|
|
|
# ---------------- Step 1: Read DEV1 hostname and sanity ----------------
|
|
- name: Read DEV1 hostname
|
|
when: nc_probe.rc == 0
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
|
|
register: dev1_host_read
|
|
changed_when: false
|
|
|
|
- name: Stop if DEV1 hostname mismatch
|
|
ansible.builtin.meta: end_host
|
|
when: (dev1_host_read.stdout | trim | length > 0) and
|
|
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
|
|
|
|
# ---------------- Step 2: Setup temporary IP for reachability ----------------
|
|
- name: Add temporary IP on DEV1 (ignore if exists)
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
register: add_ip
|
|
changed_when: add_ip.rc == 0
|
|
failed_when: >
|
|
add_ip.rc != 0 and
|
|
('File exists' not in (add_ip.stdout | default(''))) and
|
|
('File exists' not in (add_ip.stderr | default('')))
|
|
|
|
# ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
|
|
- name: Pick random free local port
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
set -e
|
|
for i in $(seq 1 25); do
|
|
p="$(shuf -i 20000-39999 -n1)"
|
|
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
|
|
done
|
|
exit 1
|
|
register: pick_port
|
|
changed_when: false
|
|
|
|
- name: Stop if no free local port
|
|
ansible.builtin.meta: end_host
|
|
when: (pick_port.stdout | trim | length) == 0
|
|
|
|
- name: Record chosen local port and control dir
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
_local_port: "{{ pick_port.stdout | trim }}"
|
|
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
|
|
|
|
- name: Build tunnel control socket path
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
|
|
|
|
- name: Start SSH ControlMaster tunnel via DEV1
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
set -e
|
|
sshpass -p '{{ dev1_pass }}' ssh -f -N \
|
|
-M -S "{{ _ctrl_sock }}" \
|
|
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
|
|
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
|
|
"{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
|
|
args:
|
|
executable: /bin/bash
|
|
register: start_tunnel
|
|
changed_when: true
|
|
|
|
# --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
|
|
- name: Small delay for tunnel to settle
|
|
delegate_to: localhost
|
|
ansible.builtin.wait_for:
|
|
timeout: 1
|
|
changed_when: false
|
|
|
|
- name: Verify tunnel master running (ssh -O check)
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
set -e
|
|
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
|
|
register: tun_check
|
|
changed_when: false
|
|
|
|
- name: Debug tunnel check
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "tunnel_check.rc={{ tun_check.rc }}"
|
|
- "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
|
|
|
|
- name: Sanity confirm tunnel TCP reachability to DEV2
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
set -e
|
|
nc -z -w5 127.0.0.1 "{{ _local_port }}"
|
|
register: nc_probe
|
|
changed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Debug reachability result
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "nc.rc={{ nc_probe.rc }}"
|
|
- "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
|
|
- "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
|
|
|
|
- name: Stop if tunnel TCP check failed
|
|
ansible.builtin.meta: end_host
|
|
when: nc_probe.rc != 0
|
|
|
|
# ---------------- Step 4: Determine working password for DEV2 ----------------
|
|
- name: Try both passfiles for DEV2
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
for f in {{ dev2_passfiles | join(' ') }}; do
|
|
if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
|
|
echo "$f"; exit 0;
|
|
fi
|
|
done
|
|
echo NONE
|
|
register: dev2_passfile_try
|
|
changed_when: false
|
|
|
|
- name: Save selected DEV2 passfile
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
|
|
changed_when: false
|
|
|
|
- name: Debug selected DEV2 passfile
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg: "dev2_passfile_used={{ dev2_passfile_used }}"
|
|
|
|
- name: Stop if no valid passfile found
|
|
ansible.builtin.meta: end_host
|
|
when: dev2_passfile_used == "NONE"
|
|
|
|
# ---------------- Step 5: Read firmware version on DEV2 ----------------
|
|
- name: Read firmware version from DEV2
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
|
|
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
|
|
root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
|
|
register: dev2_fwver
|
|
changed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Show firmware version readout
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
|
|
|
|
# ---------------- Retry metadata + success evaluation (read OK) ----------------
|
|
- name: Normalize retry metadata for indoor checker
|
|
ansible.builtin.set_fact:
|
|
attempt: "{{ (attempt | default(1)) | int }}"
|
|
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
|
|
correlation_id: "{{ correlation_id | default('') }}"
|
|
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
|
|
- name: Check if firmware read succeeded (read_ok)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
|
|
and ((dev2_fwver.stdout | default('') | trim) | length > 0)
|
|
and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
|
|
|
|
# ===================== (NEW) Unconditional normalization + comparison =====================
|
|
# Compute expected_norm from target_version_full. If the full filename-like string is sent,
|
|
# we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
|
|
|
|
- name: Normalize expected target string (step 1: compute components)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
|
|
expected_norm_core: >-
|
|
{{
|
|
(target_version_full | default('') |
|
|
regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1'))
|
|
| default('', true)
|
|
}}
|
|
|
|
- name: Normalize expected target string (step 2: choose core if present)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
|
|
|
|
# Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801"
|
|
- name: Normalize banner/firmux string from DEV2
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
|
|
banner_norm: >-
|
|
{{
|
|
(banner_raw | lower is search('-r[0-9]+$'))
|
|
| ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')))
|
|
}}
|
|
|
|
- name: Evaluate version match (normalized equality or contains)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
version_match: >-
|
|
{{
|
|
(expected_norm | length > 0)
|
|
and (
|
|
(banner_norm == expected_norm)
|
|
or (banner_norm is search(expected_norm))
|
|
or (expected_norm is search(banner_norm))
|
|
)
|
|
}}
|
|
|
|
- name: Debug compare snapshot (expected vs actual normalized)
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "expected_norm='{{ expected_norm }}'"
|
|
- "banner_norm='{{ banner_norm }}'"
|
|
- "version_match={{ version_match | default(false) }}"
|
|
|
|
# ===================== Journaling/Tagging paths =====================
|
|
# SUCCESS: read_ok AND version_match
|
|
- name: Build success journal payload
|
|
when: (read_ok | bool) and (version_match | bool)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
journal_success_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
Banner='{{ banner_raw }}' Target='{{ expected_norm }}'
|
|
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
|
|
- name: Publish success journal to control queue
|
|
when: journal_success_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_success_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_pub_success
|
|
changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
|
|
|
|
- name: Build payload to add indoor-update-success tag (DEV1)
|
|
when: journal_success_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
tag_add_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "tag_add"
|
|
task_result: "indoor-update-success"
|
|
|
|
- name: Publish indoor-update-success tag
|
|
when: tag_add_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ tag_add_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_tag_add_success
|
|
changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
|
|
|
|
- name: Build payload to remove indoor-restart-scheduled tag (DEV1)
|
|
when: journal_success_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
tag_remove_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "tag_remove"
|
|
task_result: "indoor-restart-scheduled"
|
|
|
|
- name: Publish indoor-restart-scheduled tag removal
|
|
when: tag_remove_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ tag_remove_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_tag_remove_sched
|
|
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
|
|
|
|
# --- Normalize firmware string and set custom field on success ---
|
|
- name: Capture raw firmware banner from DEV2 (for normalization)
|
|
when: journal_success_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
fw_banner_raw: "{{ banner_raw }}"
|
|
|
|
- name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
|
|
when:
|
|
- journal_success_payload is defined
|
|
- (fw_banner_raw | default('') | length) > 0
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
fw_norm: >-
|
|
{{
|
|
fw_banner_raw
|
|
if (fw_banner_raw | lower is search('-r[0-9]+$'))
|
|
else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
|
|
}}
|
|
|
|
- name: Debug normalized firmware (indoor_fwver)
|
|
when: fw_norm is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
|
|
|
|
- name: Publish custom_field_set indoor_fwver
|
|
when:
|
|
- fw_norm is defined
|
|
- (fw_norm | length) > 0
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'indoor_fwver',
|
|
'task_result': fw_norm
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_customfield_fw
|
|
changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
|
|
|
|
# MISMATCH path: firmware readable but does NOT match expected target
|
|
- name: Build mismatch journal payload
|
|
when: (read_ok | bool) and (not (version_match | bool))
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
journal_mismatch_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
|
|
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
|
|
- name: Publish mismatch journal to control queue
|
|
when: journal_mismatch_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_mismatch_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_j_mismatch
|
|
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
|
|
|
|
- name: Stop host after mismatch evaluation
|
|
when: journal_mismatch_payload is defined
|
|
ansible.builtin.meta: end_host
|
|
|
|
# FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
|
|
- name: Build failure journal payload (indoor firmware read failed)
|
|
when: not (read_ok | bool)
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
journal_fail_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
|
|
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
|
|
- name: Publish failure journal to control queue
|
|
when: journal_fail_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_fail_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_pub_fail
|
|
changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
|
|
|
|
- name: Compute retry parameters (10 minutes)
|
|
when: not (read_ok | bool)
|
|
ansible.builtin.set_fact:
|
|
next_attempt: "{{ (attempt | int) + 1 }}"
|
|
next_delay_sec: 600
|
|
next_delay_ms: "{{ 600000 }}"
|
|
|
|
- name: Build final gave-up journal (max attempts reached)
|
|
when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
journal_gaveup_payload:
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
task_name: "journal_add"
|
|
task_result: >-
|
|
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
|
|
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
|
|
- name: Publish final gave-up journal
|
|
when: journal_gaveup_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_pub_gaveup
|
|
changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
|
|
|
|
# Only schedule next attempt if we still have budget left
|
|
- name: Build delayed payload for next indoor attempt (10 min)
|
|
when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
|
|
delegate_to: localhost
|
|
ansible.builtin.set_fact:
|
|
delayed_payload:
|
|
task_name: "afterupgrade_indoor_check"
|
|
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
attempt: "{{ next_attempt | int }}"
|
|
max_attempts: "{{ effective_max_attempts | int }}"
|
|
correlation_id: "{{ correlation_id | default('') }}"
|
|
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
schema_version: 1
|
|
|
|
- name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
|
|
when: delayed_payload is defined
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
correlation_id: "{{ correlation_id | default('') }}"
|
|
routing_key: "deviceconfig"
|
|
payload: "{{ delayed_payload | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_pub_next
|
|
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
|
|
post_tasks:
|
|
- name: Cleanup tunnel and temp IP
|
|
block:
|
|
- ansible.builtin.debug:
|
|
msg: "Cleaning up tunnel + temp IP"
|
|
changed_when: false
|
|
delegate_to: localhost
|
|
always:
|
|
- name: Close tunnel
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
|
|
changed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Remove control dir
|
|
delegate_to: localhost
|
|
ansible.builtin.file:
|
|
path: "{{ _ctrl_dir | default('/tmp/none') }}"
|
|
state: absent
|
|
ignore_errors: true
|
|
|
|
- name: Remove temporary IP from DEV1
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
register: del_ip
|
|
failed_when: false
|
|
changed_when: false
|