Files
ansible-worker/files/ansible-playbooks/update-indoorbackup223r6828.yml
2026-04-01 17:39:53 +03:00

1431 lines
61 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# update-indoorbackup223r6828.yml (fork of update-indoor223-r6828.yml for backup-bank preparation without reboot/flip)
- name: Second-line indoor backup-bank preparation via DEV1 → LLDP/tunnel → DEV2 (non-invasive control path)
hosts: all
gather_facts: no
vars:
# Busybox-safe PATH prefix for all remote raw calls on DEV1
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
# Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
is_run_by_effective: "{{ is_run_by | default('manual') }}"
# DEV1 credentials (stable, like rebootin222)
dev1_user: "root"
dev1_pass: "wavewave"
# Tunnel target DEV2 behind DEV1
dev2_host: "192.168.1.1"
dev2_port: 22
# Temp IP we add to DEV1 so it can reach DEV2
dev2_side_ip: "192.168.1.11/24"
dev1_iface: "br-wan"
# Connectivity module knobs (Feb method)
arping_iface: "eth0"
ssh_timeout: 30
# DEV2 behind the tunnel
dev2_ssh_user: "root"
dev2_passfiles:
- "basicpass"
- "basicpass2"
# SSH options used from controller
ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
# Image to stage on DEV2 (we validate first; the actual write happens later)
image_filename: "fox200-2.2.3-r6828.bin"
image_md5: "f10e3cc1797cf019e740b8130cb17fc5"
# Optional SHA256; leave empty to skip SHA256 checks
image_sha256: ""
dev2_image_dir: "/tmp"
dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
# Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace).
rebootin: ""
# ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
# ---------------- New: debugging toggle for newly added debug tasks ----------------
debugging: true
# ---------------- New: abstracted DEV2 connection (decided early) ----------------
# "tunnel" (default) or "direct_lldp" or "lldp4_fallback"
dev2_conn_method: "tunnel"
dev2_ssh_host: ""
dev2_ssh_port: ""
pre_tasks:
# ------------------------------- Hostname sanity DEV1 -------------------------------
- name: Read DEV1 hostname (busybox-safe)
ansible.builtin.raw: >
{{ pathprefix }}
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
register: dev1_host_read
changed_when: false
- name: Debug incoming parameters from consumer and defaults
delegate_to: localhost
ansible.builtin.debug:
msg:
- "rebootin={{ rebootin | default('UNSET') }}"
- "image_filename={{ image_filename | default('UNSET') }}"
- "image_md5={{ image_md5 | default('UNSET') }}"
- "image_sha256={{ image_sha256 | default('UNSET') }}"
- "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
- "inventory_hostname={{ inventory_hostname }}"
- "is_run_by={{ is_run_by_effective }}"
# ====== NEW: pick up force-upgrade flag from CLI and TEMP override ======
- name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes)
ansible.builtin.set_fact:
force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | trim }}"
- name: Normalize force-upgrade flag to boolean
ansible.builtin.set_fact:
force_upgrade: "{{ (force_upgrade_raw | string | trim) | bool }}"
- name: Debug force-upgrade effective
delegate_to: localhost
ansible.builtin.debug:
msg:
- "force_upgrade_raw={{ force_upgrade_raw | default('') }}"
- "force_upgrade={{ force_upgrade | default(false) }}"
- name: Stop early if connected DEV1 hostname != inventory
ansible.builtin.meta: end_host
when: (dev1_host_read.stdout | trim | length > 0) and
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
tasks:
# ============================ CONNECTIVITY MODULE (FEB "BEST-KNOWN" METHOD) ============================
- name: Compute hostname digits key for LLDP lookup (DEV2)
ansible.builtin.set_fact:
dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
changed_when: false
- name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
ansible.builtin.raw: >
{{ pathprefix }}
DIGITS="{{ dev2_lldp_digits }}";
cat /var/run/lldp_server.json 2>/dev/null \
| grep "${DIGITS}" -A 10 \
| grep address \
| grep -vE 'subtype|ipv6' \
| awk -F'"' '{ print $4 }' \
| head -n1
register: dev2_lldp_ip_raw
changed_when: false
failed_when: false
- name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
ansible.builtin.raw: >
{{ pathprefix }}
DIGITS="{{ dev2_lldp_digits }}";
cat /var/run/lldp_server.json 2>/dev/null \
| grep "${DIGITS}" -A 15 \
| grep 'address_ipv6' \
| awk -F'"' '{ print $4 }' \
| head -n1
register: dev2_lldp_ip6_raw
changed_when: false
failed_when: false
- name: Capture LLDP-derived DEV2 IP facts
ansible.builtin.set_fact:
lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
changed_when: false
- name: Classify LLDP IPv4 candidate
delegate_to: localhost
ansible.builtin.set_fact:
lldp_ip_class: >-
{% set ip = (lldp_dev2_ip | default('')) %}
{% if ip == '' %}none
{% elif ip.startswith('10.') %}10
{% elif ip.startswith('192.168.') %}192_168
{% else %}other{% endif %}
changed_when: false
- name: Debug LLDP candidates
when: debugging | bool
delegate_to: localhost
ansible.builtin.debug:
msg:
- "LLDP digits={{ dev2_lldp_digits | default('') }}"
- "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
- "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
- "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
- name: Override dev2_host from LLDP when candidate is 192.168.x.x
when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
delegate_to: localhost
ansible.builtin.set_fact:
dev2_host: "{{ lldp_dev2_ip | trim }}"
changed_when: false
- name: Set connection method initial (direct_lldp if 10.x else tunnel)
delegate_to: localhost
ansible.builtin.set_fact:
dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
changed_when: false
- name: Initialize DEV2 passfile used
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "NONE"
changed_when: false
# ---------------------------- DIRECT LLDP IPv4 AUTH (10.x) ----------------------------
- name: Try DEV2 login via direct LLDP IPv4 (10.x) with 'basicpass'
when: dev2_conn_method == "direct_lldp"
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ lldp_dev2_ip }}"
sshpass -f basicpass ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
args:
executable: /bin/bash
register: dev2_lldp_try_basicpass
changed_when: false
ignore_errors: true
- name: Select 'basicpass' for direct LLDP if previous login succeeded
when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "basicpass"
dev2_ssh_host: "{{ lldp_dev2_ip }}"
dev2_ssh_port: 22
changed_when: false
- name: Try DEV2 login via direct LLDP IPv4 (10.x) with 'basicpass2' (only if first failed)
when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ lldp_dev2_ip }}"
sshpass -f basicpass2 ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
args:
executable: /bin/bash
register: dev2_lldp_try_basicpass2
changed_when: false
ignore_errors: true
- name: Select 'basicpass2' for direct LLDP if second login succeeded
when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE" and dev2_lldp_try_basicpass2.rc == 0
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "basicpass2"
dev2_ssh_host: "{{ lldp_dev2_ip }}"
dev2_ssh_port: 22
changed_when: false
- name: Fallback to tunnel if direct LLDP auth failed
when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE"
delegate_to: localhost
ansible.builtin.set_fact:
dev2_conn_method: "tunnel"
changed_when: false
- name: Debug connection method decision (new debug)
when: debugging | bool
delegate_to: localhost
ansible.builtin.debug:
msg:
- "dev2_conn_method={{ dev2_conn_method }}"
- "lldp_dev2_ip={{ lldp_dev2_ip | default('<none>') }}"
- "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('<none>') }}"
# ---------------------------- TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ----------------------------
- name: Add temporary IP on DEV1 (tolerate 'File exists')
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: >
{{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
register: add_ip
changed_when: add_ip.rc == 0
failed_when: >
add_ip.rc != 0
and ('File exists' not in (add_ip.stdout | default('')))
and ('File exists' not in (add_ip.stderr | default('')))
- name: Debug result of adding temp IP to DEV1
when: dev2_conn_method == "tunnel"
ansible.builtin.debug:
msg:
- "add_ip.rc={{ add_ip.rc | default('') }}"
- "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
- "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
- name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: >
{{ pathprefix }}
bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
register: dev2_mac_scan
changed_when: false
failed_when: false
- name: Capture discovered DEV2 MAC (if any)
when: dev2_conn_method == "tunnel"
ansible.builtin.set_fact:
dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
changed_when: false
- name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: >
{{ pathprefix }}
ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
register: dev2_arp_del
changed_when: false
failed_when: false
- name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
ansible.builtin.raw: >
{{ pathprefix }}
ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
register: dev2_arp_add
changed_when: dev2_arp_add.rc == 0
failed_when: >
dev2_arp_add.rc != 0
and ('File exists' not in (dev2_arp_add.stdout | default('')))
and ('File exists' not in (dev2_arp_add.stderr | default('')))
- name: Debug ARP action summary on DEV1
when: dev2_conn_method == "tunnel"
ansible.builtin.debug:
msg:
- "dev2_mac={{ dev2_mac | default('UNSET') }}"
- "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
- "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
- "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
- name: Note skipping static ARP add (no MAC discovered)
when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0)
ansible.builtin.debug:
msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
- name: Refresh ARP on DEV1s LAN (best-effort)
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: >
{{ pathprefix }}
arping -U -I {{ arping_iface }} {{ dev2_side_ip.split('/')[0] }} -c 3
changed_when: false
failed_when: false
- name: Pick a free local TCP port for the tunnel (controller side)
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.shell: |
set -e
pick() {
for i in $(seq 1 25); do
p="$(shuf -i 20000-39999 -n 1)"
if command -v ss >/dev/null 2>&1; then
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
echo "$p"; return 0
fi
else
if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
echo "$p"; return 0
fi
fi
done
return 1
}
pick
register: pick_port
changed_when: false
- name: Stop if no free local port was found
when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
ansible.builtin.meta: end_host
- name: Create control dir for SSH ControlMaster
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.shell: "mktemp -d"
args:
executable: /bin/bash
register: mktemp_dir
changed_when: false
- name: Record chosen local port and build ControlMaster socket path
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.set_fact:
_local_port: "{{ pick_port.stdout | trim }}"
_ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
_ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
changed_when: false
- name: Debug picked local port (controller)
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.debug:
msg:
- "picked_local_port={{ _local_port }}"
- "ctrl_sock={{ _ctrl_sock }}"
- "dev1_host={{ ansible_host | default(inventory_hostname) }}"
- name: Build tunnel remote target for DEV2 (prefer LLDP IPv6 when available)
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.set_fact:
dev2_tunnel_target_host: >-
{{
(lldp_dev2_ip6 is defined and (lldp_dev2_ip6 | string | trim | length > 0))
| ternary(
'[' ~ (lldp_dev2_ip6 | string | trim) ~ '%' ~ (dev1_iface | string | trim) ~ ']',
dev2_host | string
)
}}
dev2_tunnel_target_port: "{{ dev2_port | string }}"
changed_when: false
- name: Debug tunnel remote target for DEV2 (IPv4/IPv6)
when: dev2_conn_method == "tunnel" and debugging | bool
delegate_to: localhost
ansible.builtin.debug:
msg:
- "dev2_tunnel_target={{ dev2_tunnel_target_host | default(dev2_host) }}:{{ dev2_tunnel_target_port | default(dev2_port) }}"
- name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.shell: |
set -e
USER="{{ dev1_user }}"
HOST="{{ ansible_host | default(inventory_hostname) }}"
sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
-M -S "{{ _ctrl_sock }}" \
-L "127.0.0.1:{{ _local_port }}:{{ dev2_tunnel_target_host | default(dev2_host) }}:{{ dev2_tunnel_target_port | default(dev2_port) }}" \
"${USER}@${HOST}"
args:
executable: /bin/bash
register: start_tunnel
changed_when: true
- name: Probe TCP reachability to DEV2 through the tunnel (nc)
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.shell: |
set -e
nc -z -w5 127.0.0.1 "{{ _local_port }}"
register: nc_probe
changed_when: false
ignore_errors: true
- name: Stop if tunnel TCP probe failed
when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
ansible.builtin.meta: end_host
- name: Show passfiles available on controller (ls)
delegate_to: localhost
ansible.builtin.shell: |
set -e
ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
register: dev2_ls
changed_when: false
- name: Debug passfiles presence
delegate_to: localhost
ansible.builtin.debug:
msg:
- "{{ (dev2_ls.stdout | default('')) | trim }}"
- "{{ (dev2_ls.stderr | default('')) | trim }}"
- name: Try DEV2 login with 'basicpass' (root, tunnel)
when: dev2_conn_method == "tunnel"
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f basicpass ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
args:
executable: /bin/bash
register: dev2_try_basicpass
changed_when: false
ignore_errors: true
- name: Select 'basicpass' if previous login succeeded (tunnel)
when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "basicpass"
dev2_ssh_host: "127.0.0.1"
dev2_ssh_port: "{{ _local_port }}"
changed_when: false
- name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel)
when: dev2_conn_method == "tunnel" and dev2_passfile_used == "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f basicpass2 ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
args:
executable: /bin/bash
register: dev2_try_basicpass2
changed_when: false
ignore_errors: true
- name: Select 'basicpass2' if second login succeeded (tunnel)
when: dev2_conn_method == "tunnel" and dev2_passfile_used == "NONE" and dev2_try_basicpass2.rc == 0
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "basicpass2"
dev2_ssh_host: "127.0.0.1"
dev2_ssh_port: "{{ _local_port }}"
changed_when: false
# ---------------------------- FALLBACKS (Feb method style) ----------------------------
- name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed)
when:
- dev2_conn_method == "tunnel"
- dev2_passfile_used == "NONE"
- (lldp_dev2_ip | default('')) is match('^10\\.')
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ lldp_dev2_ip }}"
if sshpass -f basicpass ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=20 \
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
echo "basicpass"; exit 0
fi
if sshpass -f basicpass2 ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=20 \
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
echo "basicpass2"; exit 0
fi
echo "NONE"; exit 1
args:
executable: /bin/bash
register: dev2_auth_lldp4
changed_when: false
failed_when: false
ignore_errors: true
- name: Apply LLDP IPv4 fallback decision
when:
- dev2_conn_method == "tunnel"
- dev2_passfile_used == "NONE"
- (lldp_dev2_ip | default('')) is match('^10\\.')
- dev2_auth_lldp4 is defined
- (dev2_auth_lldp4.stdout | trim) in ['basicpass','basicpass2']
delegate_to: localhost
ansible.builtin.set_fact:
dev2_conn_method: "lldp4_fallback"
dev2_passfile_used: "{{ dev2_auth_lldp4.stdout | trim }}"
dev2_ssh_host: "{{ lldp_dev2_ip }}"
dev2_ssh_port: 22
changed_when: false
- name: Debug final DEV2 connectivity decision (upgrade will use host/port/passfile below)
when: debugging | bool
delegate_to: localhost
ansible.builtin.debug:
msg:
- "dev2_conn_method={{ dev2_conn_method }}"
- "dev2_passfile_used={{ dev2_passfile_used }}"
- "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
- "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
- "lldp_dev2_ip={{ lldp_dev2_ip | default('') }}"
- "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('') }}"
- "dev2_host={{ dev2_host }}:{{ dev2_port }}"
# ============================ BACKUP PREPARATION PROCEDURE BELOW (SURGICALLY FORKED) ============================
# ---------------------------- Read DEV2 hostname via chosen path ----------------------------
- name: Read DEV2 hostname via selected connection (busybox-safe)
when: dev2_passfile_used != "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
args:
executable: /bin/bash
register: dev2_host_read
changed_when: false
- name: Normalize hostnames for strict compare (inventory/DEV1/DEV2)
ansible.builtin.set_fact:
_inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
_dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
_dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
- name: Debug normalized hostnames (JSON-escaped)
delegate_to: localhost
ansible.builtin.debug:
msg:
- "inv={{ _inv_hn | tojson }}"
- "dev1={{ _dev1_hn | tojson }}"
- "dev2={{ _dev2_hn | tojson }}"
# -------- New: DEV2 hostname classification (numeric ikejaNNNN vs other) --------
- name: Detect DEV2 hostname pattern (ikeja<number> or other) and raw form
ansible.builtin.set_fact:
_dev2_hn_is_ikeja_numeric: "{{ (_dev2_hn | default('')) is match('^ikeja[0-9]+$') }}"
_dev2_hn_raw: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') }}"
changed_when: false
- name: Classify DEV2 hostname mismatch type
ansible.builtin.set_fact:
_hostname_softfail: "{{ _dev2_hn_is_ikeja_numeric and (_dev2_hn != _inv_hn) }}"
_hostname_warn: "{{ (not _dev2_hn_is_ikeja_numeric) and (_dev2_hn != _inv_hn) }}"
changed_when: false
- name: Debug DEV2 hostname classification
delegate_to: localhost
ansible.builtin.debug:
msg:
- "_dev2_hn_is_ikeja_numeric={{ _dev2_hn_is_ikeja_numeric | default(false) }}"
- "_hostname_softfail={{ _hostname_softfail | default(false) }}"
- "_hostname_warn={{ _hostname_warn | default(false) }}"
- "_dev2_hn_raw={{ _dev2_hn_raw | default('') }}"
# ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
- name: Init soft-fail journal flags
ansible.builtin.set_fact:
_journal: []
_prep_blocked: false
_blocked: false
- name: Append connection method decision to journal
ansible.builtin.set_fact:
_journal: "{{ _journal + [ 'Connection method for DEV2: ' ~ dev2_conn_method ~ (((dev2_conn_method == 'direct_lldp') | ternary(' (' ~ (dev2_ssh_host | default('')) ~ ')',''))) ] }}"
# ---- Apply hostname policy: soft-fail for other ikejaNNNN, warn+customfield for "weird" names ----
- name: Soft-block if DEV2 hostname is another ikeja<number> (probable wrong indoor)
when: _hostname_softfail | default(false)
ansible.builtin.set_fact:
_blocked: true
_journal: "{{ (_journal | default([])) + [ 'DEV2 hostname looks like another ikeja-number device: dev2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ '. Treating as wrong indoor, skipping update.' ] }}"
- name: Note non-standard DEV2 hostname and prepare indoor_hostname customfield
when: _hostname_warn | default(false)
ansible.builtin.set_fact:
indoor_hostname_value: "{{ '!' ~ (_dev2_hn_raw | default(_dev2_hn)) }}"
_journal: "{{ (_journal | default([])) + [ 'DEV2 hostname non-standard: ' ~ _dev2_hn_raw ~ ' (customfield indoor_hostname=' ~ ('!' ~ (_dev2_hn_raw | default(_dev2_hn))) ~ ')' ] }}"
- name: Build indoor_hostname customfield payload
when: _hostname_warn | default(false)
delegate_to: localhost
ansible.builtin.set_fact:
indoor_hostname_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "customfield_set"
task_result: "indoor_hostname={{ indoor_hostname_value }}"
- name: Publish indoor_hostname custom field to control queue
delegate_to: localhost
uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ {
'inscope_device': (ansible_hostname | default(inventory_hostname)),
'task_name': 'custom_field_set',
'task_add1': 'indoor_hostname',
'task_result': indoor_hostname_value
} | to_json }}"
payload_encoding: "string"
changed_when: false
when: indoor_hostname_value is defined
- name: Build specific backup marker path on DEV2 (/root/prepared_backup_<image>)
ansible.builtin.set_fact:
_marker_specific: "/root/prepared_backup_{{ image_filename | regex_replace('\\.bin$','') }}"
- name: Init backup marker state flags
ansible.builtin.set_fact:
_backup_marker_created: false
_backup_prepare_success: false
- name: Check whether specific backup marker already exists on DEV2
when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh -o AddressFamily=inet -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 -o ConnectTimeout=10 -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "[ -f '{{ _marker_specific }}' ] && echo PRESENT || echo ABSENT"
args:
executable: /bin/bash
register: dev2_backup_marker_present
changed_when: false
ignore_errors: true
- name: Soft-block if specific backup marker already exists on DEV2
when: dev2_backup_marker_present is defined
and (dev2_backup_marker_present.stdout | default('') | trim) == 'PRESENT'
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Backup marker already present on DEV2 (' ~ _marker_specific ~ '). Skipping backup preparation/write' ] }}"
- name: Create specific backup marker on DEV2 before preparation starts
when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 -o ConnectTimeout=10 -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "touch '{{ _marker_specific }}'"
args:
executable: /bin/bash
register: dev2_backup_marker_create
changed_when: true
- name: Record that backup marker was created by this run
when: dev2_backup_marker_create is defined and (dev2_backup_marker_create.rc | default(1) | int) == 0
ansible.builtin.set_fact:
_backup_marker_created: true
- name: Check bootenv-updated marker on DEV2 (/tmp/bootenv_updated)
when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"[ -f '/tmp/bootenv_updated' ] && echo PRESENT || echo ABSENT"
args:
executable: /bin/bash
register: dev2_bootenv_marker
changed_when: false
ignore_errors: true
- name: Soft-block if bootenv-updated marker is present on DEV2
when: dev2_bootenv_marker is defined
and (dev2_bootenv_marker.stdout | default('') | trim) == 'PRESENT'
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Bootenv updated marker present on DEV2 (/tmp/bootenv_updated). Skipping image staging/write' ] }}"
- name: Refresh ARP 1 on DEV1s LAN (send unsolicited ARP from temporary IP)
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
# ---------------------------- DEV2 version firmux primary check ----------------------------
- name: Read DEV2 /usr/lib/release/firmux (if present)
when: dev2_passfile_used != "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"cat /usr/lib/release/firmux 2>/dev/null || true"
args:
executable: /bin/bash
register: dev2_firmux
changed_when: false
ignore_errors: true
- name: Debug DEV2 firmux banner (if any)
when: dev2_firmux is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
# ====== INSERTED: two-step normalization preview (rev->r, then space->dash) ======
- name: "Normalize DEV2 firmux (step 1): replace 'rev ' -> 'r' (debug only)"
when: dev2_firmux is defined
delegate_to: localhost
ansible.builtin.set_fact:
_firmux_step1: "{{ (dev2_firmux.stdout | default('') | trim) | regex_replace('(?i)rev\\s+','r') }}"
changed_when: false
- name: "Debug firmux after step 1 (rev->r)"
when: _firmux_step1 is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "_firmux_step1={{ _firmux_step1 }}"
- name: "Normalize DEV2 firmux (step 2): replace space -> '-' (debug only)"
when: _firmux_step1 is defined
delegate_to: localhost
ansible.builtin.set_fact:
_firmux_step2: "{{ _firmux_step1 | replace(' ', '-') }}"
changed_when: false
- name: "Debug firmux after step 2 (space->dash)"
when: _firmux_step2 is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "_firmux_step2={{ _firmux_step2 }}"
# ====== END INSERTED ======
# ===================== Early equality/substring check (same logic as checker) =====================
- name: Normalize expected target step one compute base string (from image_filename)
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_step1: "{{ (image_filename | default('') | trim) }}"
changed_when: false
- name: Extract version core X dot Y dot Z dash rNNNN from image_filename if present
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_core_list: "{{ (image_filename | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
changed_when: false
- name: Choose first extracted core if available
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
changed_when: false
- name: Normalize expected target step two prefer extracted core when available
delegate_to: localhost
ansible.builtin.set_fact:
expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
changed_when: false
- name: Capture firmware banner line from DEV2 (raw)
delegate_to: localhost
ansible.builtin.set_fact:
banner_raw: "{{ (dev2_firmux.stdout | default('') | trim) }}"
changed_when: false
- name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
delegate_to: localhost
ansible.builtin.set_fact:
banner_norm: >-
{{
((banner_raw | lower) is search('-r[0-9]+$'))
| ternary(
banner_raw,
(banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1'))
)
}}
changed_when: false
- name: Evaluate version match using normalized equality or substring
delegate_to: localhost
ansible.builtin.set_fact:
version_match: >-
{{
(expected_norm | default('') | length > 0)
and (
(banner_norm | default('')) == (expected_norm | default(''))
or ((banner_norm | default('') | lower) is search((expected_norm | default('') | lower)))
or ((expected_norm | default('') | lower) is search((banner_norm | default('') | lower)))
)
}}
changed_when: false
- name: Debug version compare snapshot (pre-write)
delegate_to: localhost
ansible.builtin.debug:
msg:
- "expected_norm={{ expected_norm | default('') }}"
- "banner_raw={{ banner_raw | default('') }}"
- "banner_norm={{ banner_norm | default('') }}"
- "force_upgrade={{ force_upgrade | default(false) }}"
- "version_match={{ version_match | default(false) }}"
- name: Back-fill compare names for summary (no-op if already set)
when: not (_blocked | default(false))
ansible.builtin.set_fact:
expected_fw_core_early: "{{ expected_fw_core_early | default(expected_norm | default('')) }}"
current_fw_core: "{{ current_fw_core | default(banner_norm | default('')) }}"
changed_when: false
- name: Check local presence of image file on controller
delegate_to: localhost
ansible.builtin.stat:
path: "{{ image_filename }}"
register: local_img
- name: Soft-block if local image is missing
when: not local_img.stat.exists
ansible.builtin.set_fact:
_blocked: true
_journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
- name: Compute local md5 of the image (controller)
when: local_img.stat.exists
delegate_to: localhost
ansible.builtin.shell: |
set -e
md5sum "{{ image_filename }}" | awk '{print $1}'
register: local_md5
changed_when: false
- name: Verify local md5 matches expected
when: local_img.stat.exists
delegate_to: localhost
ansible.builtin.assert:
that:
- (local_md5.stdout | trim) == image_md5
fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
success_msg: "Local md5 matches expected"
- name: Compute local sha256 of the image (if provided)
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
sha256sum "{{ image_filename }}" | awk '{print $1}'
register: local_sha256
changed_when: false
ignore_errors: true
- name: Soft-block if local sha256 mismatch/unavailable
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
- name: Refresh ARP 3 on DEV1s LAN (send unsolicited ARP from temporary IP)
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
- name: Read fw_printenv size (line count) on DEV2 (soft health)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"(fw_printenv 2>/dev/null | wc -l) || echo 0"
args:
executable: /bin/bash
register: dev2_fwenv_wc
changed_when: false
ignore_errors: true
- name: Soft-block if fw_printenv too small (<30 lines)
when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
- name: Check existing DEV2 image md5 (NOFILE if missing)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
args:
executable: /bin/bash
register: dev2_md5_before
changed_when: false
- name: Copy image to DEV2 if missing or md5 mismatch
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" scp \
-P "$PORT" \
-O \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
"{{ image_filename }}" "{{ dev2_ssh_user }}@${HOST}:{{ (dev2_ssh_user == 'root') | ternary('/', '') }}{{ dev2_image_dir }}/{{ image_filename }}"
args:
executable: /bin/bash
register: scp_push
changed_when: true
- name: Compute md5 of image on DEV2 after copy (or if already present)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
args:
executable: /bin/bash
register: dev2_md5_after
changed_when: false
- name: Soft-block if DEV2 md5 != expected
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
- name: Compute sha256 of image on DEV2 (if provided)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
args:
executable: /bin/bash
register: dev2_sha256_after
changed_when: false
ignore_errors: true
- name: Soft-block if DEV2 sha256 mismatch/unavailable
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
- name: Pre-validate image on DEV2 with 'update -c' (non-invasive)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=30 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"update -c '{{ dev2_image_path }}' 2>&1 || true"
args:
executable: /bin/bash
register: dev2_update_check
changed_when: false
- name: Debug output from 'update -c'
when: dev2_update_check is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
- name: Soft-block if 'update -c' did not return 'valid'
when: dev2_update_check is defined and not (_blocked | default(false)) and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
# ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
- name: Build control queue payload for 'indoor aborted' journal
ansible.builtin.set_fact:
journal_indoor_aborted:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
indoor backup preparation aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
when: (_blocked | default(false))
delegate_to: localhost
- name: Publish 'indoor aborted' journal to control queue
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_indoor_aborted | to_json }}"
payload_encoding: "string"
register: rmq_journal_indoor_aborted_resp
changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
failed_when: >
(rmq_journal_indoor_aborted_resp.status != 200) or
(rmq_journal_indoor_aborted_resp.json is not defined) or
(not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
when: journal_indoor_aborted is defined
delegate_to: localhost
- name: Refresh ARP 1 on DEV1s LAN (send unsolicited ARP from temporary IP)
when: dev2_conn_method == "tunnel"
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
# ============================ ACTUAL BACKUP WRITE (no reboot, no bank flip; only if not blocked) ============================
- name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
when: not (_blocked | default(false))
block:
# --- supervised async write with stall handling ---
- name: Launch update -w on DEV2 (async)
delegate_to: localhost
vars:
_write_async_cap: 600 # 10 minutes max runtime (tune if needed)
_write_poll: 20 # poll every 20 seconds
block:
- name: Launch update -w on DEV2 (async)
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=0 \
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
"update -w '{{ dev2_image_path }}'"
args:
executable: /bin/bash
async: "{{ _write_async_cap }}"
poll: "{{ _write_poll }}"
register: dev2_up_write
changed_when: true
failed_when: false
- name: Evaluate async result
ansible.builtin.set_fact:
_updatew_stalled: >-
{{ (dev2_up_write.msg is defined) and
('did not complete' in dev2_up_write.msg | lower) }}
_updatew_success: >-
{{
((dev2_up_write.stdout | default('')) is search('update is complete'))
or
((not (dev2_up_write.skipped | default(false))) and ((((dev2_up_write.stdout | default('')) | trim) == '') and (((dev2_up_write.rc | default(1)) | int) == 0)))
}}
- name: Journal stall if update -w timed out
when: _updatew_stalled | bool
ansible.builtin.set_fact:
journal_updatew_stalled:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: "update -w has stalled during backup preparation. Marker will be removed; please schedule another backup preparation."
- name: Publish stall journal to control queue
when: journal_updatew_stalled is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_updatew_stalled | to_json }}"
payload_encoding: "string"
register: rmq_stall_journal_resp
changed_when: (rmq_stall_journal_resp.json is defined) and (rmq_stall_journal_resp.json.routed | default(false) | bool)
failed_when: false
- name: Stop play if update -w stalled
when: _updatew_stalled | bool
ansible.builtin.meta: end_play
- name: Fail play if update -w completed but invalid output
when: (not _updatew_stalled) and (not _updatew_success)
ansible.builtin.fail:
msg: "update -w finished but did not report 'update is complete'"
# --- continue normal path if not stalled ---
- name: Mark backup preparation success after update -w
when: not _updatew_stalled | bool and (_updatew_success | default(false) | bool)
ansible.builtin.set_fact:
_backup_prepare_success: true
# ---------------------------- No reboot scheduling in backup fork ----------------------------
# ---------------------------- Journal: indoor backup prepared ----------------------------
- name: Compute write success flag (_write_success)
ansible.builtin.set_fact:
_write_success: >-
{{
(
(
(dev2_up_write.stdout | default('')) is search('update is complete')
)
or
(
((not (dev2_up_write.skipped | default(false))) and (((dev2_up_write.stdout | default('')) | trim) == '')
and (((dev2_up_write.rc | default(1)) | int) == 0))
)
)
if (dev2_up_write is defined) else false
}}
delegate_to: localhost
- name: Debug write result and backup marker (quick summary)
when: not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.debug:
msg:
- "write_success={{ _write_success | default(false) }}"
- "backup_marker={{ _marker_specific | default('NA') }}"
- "backup_prepare_success={{ _backup_prepare_success | default(false) }}"
- name: Build 'indoor backup prepared' journal payload text
ansible.builtin.set_fact:
journal_indoor_updated:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
Indoor backup partition prepared successfully.
conn_method={{ dev2_conn_method }},
write_done={{ _write_success }},
marker={{ _marker_specific }}
when: (_write_success | bool)
delegate_to: localhost
- name: Publish 'indoor backup prepared' journal to control queue
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_indoor_updated | to_json }}"
payload_encoding: "string"
register: rmq_journal_indoor_updated_resp
changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
failed_when: >
(rmq_journal_indoor_updated_resp.status != 200) or
(rmq_journal_indoor_updated_resp.json is not defined) or
(not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
when: journal_indoor_updated is defined
delegate_to: localhost
# ---------------------------- Success follow-up scheduling ----------------------------
- name: Build sot-updater-scheduler payload after successful backup preparation
ansible.builtin.set_fact:
sot_updater_scheduler_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "sot-updater-scheduler"
when: not (_blocked | default(false)) and (_write_success | bool)
delegate_to: localhost
- name: Publish sot-updater-scheduler payload to deviceconfig
when: sot_updater_scheduler_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "deviceconfig"
payload: "{{ sot_updater_scheduler_payload | to_json }}"
payload_encoding: "string"
changed_when: false
# ---------------------------- Final operator summary (one-liners) ----------------------------
- name: Summary key outcomes (one-liners)
delegate_to: localhost
ansible.builtin.debug:
msg:
- "dev2_conn_method={{ dev2_conn_method }}"
- "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
- "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
- "dev2_passfile_used={{ dev2_passfile_used }}"
- "dev2_hostname_raw={{ _dev2_hn_raw | default('') }}"
- "hostname_softfail={{ _hostname_softfail | default(false) }}"
- "hostname_warn={{ _hostname_warn | default(false) }}"
- "indoor_hostname_value={{ indoor_hostname_value | default('') }}"
- "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
- "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
- "current_fw_core={{ current_fw_core | default('') }}"
- "force_upgrade={{ force_upgrade | default(false) }}"
- "local_image_present={{ local_img.stat.exists | default(false) }}"
- "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
- "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
- "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
- "update_c_len={{ (dev2_update_check.stdout | default('') ) | length if (dev2_update_check is defined) else 'NA' }}"
- "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
- "backup_marker={{ _marker_specific | default('') }}"
- "backup_marker_created={{ _backup_marker_created | default(false) }}"
- "backup_prepare_success={{ _backup_prepare_success | default(false) }}"
- "prep_blocked={{ _prep_blocked | default(false) }}"
- "blocked={{ _blocked | default(false) }}"
- "journal={{ (_journal | default([])) | join(' || ') }}"
post_tasks:
- name: Cleanup (always)
block:
- ansible.builtin.debug:
msg: "Entering cleanup block"
changed_when: false
delegate_to: localhost
always:
- name: Close SSH ControlMaster (best-effort)
delegate_to: localhost
ansible.builtin.shell: |
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
changed_when: false
ignore_errors: true
- name: Remove tunnel control dir (best-effort)
delegate_to: localhost
ansible.builtin.file:
path: "{{ _ctrl_dir | default('/tmp/none') }}"
state: absent
ignore_errors: true
- name: Remove backup marker on DEV2 when backup preparation did not succeed
when: (_backup_marker_created | default(false)) and not (_backup_prepare_success | default(false)) and (dev2_passfile_used | default('NONE')) != 'NONE'
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ dev2_ssh_host }}"
PORT="{{ dev2_ssh_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 -o ConnectTimeout=10 -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "rm -f '{{ _marker_specific }}'"
args:
executable: /bin/bash
changed_when: true
ignore_errors: true
- name: Debug backup marker cleanup result
when: (_backup_marker_created | default(false)) and not (_backup_prepare_success | default(false))
delegate_to: localhost
ansible.builtin.debug:
msg:
- "backup_marker_cleanup_attempted=True"
- "backup_marker={{ _marker_specific | default('') }}"
changed_when: false
- name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address' and 'Address not available')
when: dev2_conn_method | default('') == "tunnel"
ansible.builtin.raw: >
{{ pathprefix }}
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
register: del_ip
changed_when: del_ip.rc == 0
failed_when: >
del_ip.rc != 0
and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
and ('Address not available' not in (del_ip.stdout | default('')))
and ('Address not available' not in (del_ip.stderr | default('')))
and ('RTNETLINK answers: Address not available' not in (del_ip.stdout | default('')))
and ('RTNETLINK answers: Address not available' not in (del_ip.stderr | default('')))
- name: Debug temp IP removal result
ansible.builtin.debug:
msg:
- "del_ip.rc={{ del_ip.rc | default('') }}"
- "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
- "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
when: del_ip is defined