1812 lines
78 KiB
YAML
1812 lines
78 KiB
YAML
# update-indoor6-stable.yml (Indoor5-stable + surgical update-w success/journaling fixes)
|
||
- name: Second-line indoor upgrade via DEV1 → LLDP/tunnel → DEV2 (non-invasive control path)
|
||
hosts: all
|
||
gather_facts: no
|
||
|
||
vars:
|
||
# Busybox-safe PATH prefix for all remote raw calls on DEV1
|
||
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
||
|
||
# Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
|
||
is_run_by_effective: "{{ is_run_by | default('manual') }}"
|
||
|
||
# DEV1 credentials (stable, like rebootin222)
|
||
dev1_user: "root"
|
||
dev1_pass: "wavewave"
|
||
|
||
# Tunnel target DEV2 behind DEV1
|
||
dev2_host: "192.168.1.1"
|
||
dev2_port: 22
|
||
|
||
# Temp IP we add to DEV1 so it can reach DEV2
|
||
dev2_side_ip: "192.168.1.11/24"
|
||
dev1_iface: "br-wan"
|
||
|
||
# Connectivity module knobs (Feb method)
|
||
arping_iface: "eth0"
|
||
ssh_timeout: 30
|
||
|
||
# DEV2 behind the tunnel
|
||
dev2_ssh_user: "root"
|
||
dev2_passfiles:
|
||
- "basicpass"
|
||
- "basicpass2"
|
||
|
||
# SSH options used from controller
|
||
ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
|
||
|
||
# Image to stage on DEV2 (we validate first; the actual write happens later)
|
||
image_filename: "fox200-2.2.3-r6828.bin"
|
||
image_md5: "f10e3cc1797cf019e740b8130cb17fc5"
|
||
# Optional SHA256; leave empty to skip SHA256 checks
|
||
image_sha256: ""
|
||
dev2_image_dir: "/tmp"
|
||
dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
|
||
|
||
# Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace).
|
||
rebootin: ""
|
||
|
||
# ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
|
||
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
||
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
||
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
||
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
||
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
||
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
||
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
||
afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
|
||
|
||
# ---------------- New: debugging toggle for newly added debug tasks ----------------
|
||
debugging: true
|
||
|
||
# ---------------- New: abstracted DEV2 connection (decided early) ----------------
|
||
# "tunnel" (default) or "direct_lldp" or "lldp4_fallback"
|
||
dev2_conn_method: "tunnel"
|
||
dev2_ssh_host: ""
|
||
dev2_ssh_port: ""
|
||
|
||
pre_tasks:
|
||
# ------------------------------- Hostname sanity DEV1 -------------------------------
|
||
- name: Read DEV1 hostname (busybox-safe)
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
|
||
register: dev1_host_read
|
||
changed_when: false
|
||
|
||
- name: Debug incoming parameters from consumer and defaults
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "rebootin={{ rebootin | default('UNSET') }}"
|
||
- "image_filename={{ image_filename | default('UNSET') }}"
|
||
- "image_md5={{ image_md5 | default('UNSET') }}"
|
||
- "image_sha256={{ image_sha256 | default('UNSET') }}"
|
||
- "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
|
||
- "inventory_hostname={{ inventory_hostname }}"
|
||
- "is_run_by={{ is_run_by_effective }}"
|
||
|
||
# ====== NEW: pick up force-upgrade flag from CLI and TEMP override ======
|
||
- name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes)
|
||
ansible.builtin.set_fact:
|
||
force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | trim }}"
|
||
|
||
# >>> TEMPORARY OVERRIDE (REMOVE THIS TASK LATER) <<<
|
||
- name: TEMPORARILY set force-upgrade to yes (REMOVE BEFORE COMMITTING)
|
||
ansible.builtin.set_fact:
|
||
force_upgrade_raw: "no"
|
||
# <<< END TEMPORARY >>>
|
||
|
||
- name: Normalize force-upgrade flag to boolean
|
||
ansible.builtin.set_fact:
|
||
force_upgrade: "{{ (force_upgrade_raw | string | trim) | bool }}"
|
||
|
||
- name: Debug force-upgrade effective
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "force_upgrade_raw={{ force_upgrade_raw | default('') }}"
|
||
- "force_upgrade={{ force_upgrade | default(false) }}"
|
||
|
||
- name: Stop early if connected DEV1 hostname != inventory
|
||
ansible.builtin.meta: end_host
|
||
when: (dev1_host_read.stdout | trim | length > 0) and
|
||
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
|
||
|
||
tasks:
|
||
# ============================ CONNECTIVITY MODULE (FEB "BEST-KNOWN" METHOD) ============================
|
||
|
||
- name: Compute hostname digits key for LLDP lookup (DEV2)
|
||
ansible.builtin.set_fact:
|
||
dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
|
||
changed_when: false
|
||
|
||
- name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
DIGITS="{{ dev2_lldp_digits }}";
|
||
cat /var/run/lldp_server.json 2>/dev/null \
|
||
| grep "${DIGITS}" -A 10 \
|
||
| grep address \
|
||
| grep -vE 'subtype|ipv6' \
|
||
| awk -F'"' '{ print $4 }' \
|
||
| head -n1
|
||
register: dev2_lldp_ip_raw
|
||
changed_when: false
|
||
failed_when: false
|
||
|
||
- name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
DIGITS="{{ dev2_lldp_digits }}";
|
||
cat /var/run/lldp_server.json 2>/dev/null \
|
||
| grep "${DIGITS}" -A 15 \
|
||
| grep 'address_ipv6' \
|
||
| awk -F'"' '{ print $4 }' \
|
||
| head -n1
|
||
register: dev2_lldp_ip6_raw
|
||
changed_when: false
|
||
failed_when: false
|
||
|
||
- name: Capture LLDP-derived DEV2 IP facts
|
||
ansible.builtin.set_fact:
|
||
lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
|
||
lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
|
||
changed_when: false
|
||
|
||
- name: Classify LLDP IPv4 candidate
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
lldp_ip_class: >-
|
||
{% set ip = (lldp_dev2_ip | default('')) %}
|
||
{% if ip == '' %}none
|
||
{% elif ip.startswith('10.') %}10
|
||
{% elif ip.startswith('192.168.') %}192_168
|
||
{% else %}other{% endif %}
|
||
changed_when: false
|
||
|
||
- name: Debug LLDP candidates
|
||
when: debugging | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "LLDP digits={{ dev2_lldp_digits | default('') }}"
|
||
- "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
|
||
- "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
|
||
- "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
|
||
|
||
- name: Override dev2_host from LLDP when candidate is 192.168.x.x
|
||
when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_host: "{{ lldp_dev2_ip | trim }}"
|
||
changed_when: false
|
||
|
||
- name: Set connection method initial (direct_lldp if 10.x else tunnel)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
|
||
changed_when: false
|
||
|
||
- name: Initialize DEV2 passfile used
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_passfile_used: "NONE"
|
||
changed_when: false
|
||
|
||
# ---------------------------- DIRECT LLDP IPv4 AUTH (10.x) ----------------------------
|
||
- name: Try DEV2 login via direct LLDP IPv4 (10.x) with 'basicpass'
|
||
when: dev2_conn_method == "direct_lldp"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ lldp_dev2_ip }}"
|
||
sshpass -f basicpass ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_lldp_try_basicpass
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Select 'basicpass' for direct LLDP if previous login succeeded
|
||
when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_passfile_used: "basicpass"
|
||
dev2_ssh_host: "{{ lldp_dev2_ip }}"
|
||
dev2_ssh_port: 22
|
||
changed_when: false
|
||
|
||
- name: Try DEV2 login via direct LLDP IPv4 (10.x) with 'basicpass2' (only if first failed)
|
||
when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ lldp_dev2_ip }}"
|
||
sshpass -f basicpass2 ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_lldp_try_basicpass2
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Select 'basicpass2' for direct LLDP if second login succeeded
|
||
when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE" and dev2_lldp_try_basicpass2.rc == 0
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_passfile_used: "basicpass2"
|
||
dev2_ssh_host: "{{ lldp_dev2_ip }}"
|
||
dev2_ssh_port: 22
|
||
changed_when: false
|
||
|
||
- name: Fallback to tunnel if direct LLDP auth failed
|
||
when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE"
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_conn_method: "tunnel"
|
||
changed_when: false
|
||
|
||
- name: Debug connection method decision (new debug)
|
||
when: debugging | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "dev2_conn_method={{ dev2_conn_method }}"
|
||
- "lldp_dev2_ip={{ lldp_dev2_ip | default('<none>') }}"
|
||
- "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('<none>') }}"
|
||
|
||
# ---------------------------- TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ----------------------------
|
||
- name: Add temporary IP on DEV1 (tolerate 'File exists')
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
|
||
register: add_ip
|
||
changed_when: add_ip.rc == 0
|
||
failed_when: >
|
||
add_ip.rc != 0
|
||
and ('File exists' not in (add_ip.stdout | default('')))
|
||
and ('File exists' not in (add_ip.stderr | default('')))
|
||
|
||
- name: Debug result of adding temp IP to DEV1
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "add_ip.rc={{ add_ip.rc | default('') }}"
|
||
- "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
|
||
- "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
|
||
|
||
- name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
|
||
register: dev2_mac_scan
|
||
changed_when: false
|
||
failed_when: false
|
||
|
||
- name: Capture discovered DEV2 MAC (if any)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.set_fact:
|
||
dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
|
||
changed_when: false
|
||
|
||
- name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
|
||
register: dev2_arp_del
|
||
changed_when: false
|
||
failed_when: false
|
||
|
||
- name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
|
||
when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
|
||
register: dev2_arp_add
|
||
changed_when: dev2_arp_add.rc == 0
|
||
failed_when: >
|
||
dev2_arp_add.rc != 0
|
||
and ('File exists' not in (dev2_arp_add.stdout | default('')))
|
||
and ('File exists' not in (dev2_arp_add.stderr | default('')))
|
||
|
||
- name: Debug ARP action summary on DEV1
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "dev2_mac={{ dev2_mac | default('UNSET') }}"
|
||
- "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
|
||
- "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
|
||
- "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
|
||
|
||
- name: Note skipping static ARP add (no MAC discovered)
|
||
when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0)
|
||
ansible.builtin.debug:
|
||
msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
|
||
|
||
- name: Refresh ARP on DEV1’s LAN (best-effort)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
arping -U -I {{ arping_iface }} {{ dev2_side_ip.split('/')[0] }} -c 3
|
||
changed_when: false
|
||
failed_when: false
|
||
|
||
- name: Pick a free local TCP port for the tunnel (controller side)
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
pick() {
|
||
for i in $(seq 1 25); do
|
||
p="$(shuf -i 20000-39999 -n 1)"
|
||
if command -v ss >/dev/null 2>&1; then
|
||
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
|
||
echo "$p"; return 0
|
||
fi
|
||
else
|
||
if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
|
||
echo "$p"; return 0
|
||
fi
|
||
fi
|
||
done
|
||
return 1
|
||
}
|
||
pick
|
||
register: pick_port
|
||
changed_when: false
|
||
|
||
- name: Stop if no free local port was found
|
||
when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
|
||
ansible.builtin.meta: end_host
|
||
|
||
- name: Create control dir for SSH ControlMaster
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: "mktemp -d"
|
||
args:
|
||
executable: /bin/bash
|
||
register: mktemp_dir
|
||
changed_when: false
|
||
|
||
- name: Record chosen local port and build ControlMaster socket path
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
_local_port: "{{ pick_port.stdout | trim }}"
|
||
_ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
|
||
_ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
|
||
changed_when: false
|
||
|
||
- name: Debug picked local port (controller)
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "picked_local_port={{ _local_port }}"
|
||
- "ctrl_sock={{ _ctrl_sock }}"
|
||
- "dev1_host={{ ansible_host | default(inventory_hostname) }}"
|
||
|
||
- name: Build tunnel remote target for DEV2 (prefer LLDP IPv6 when available)
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_tunnel_target_host: >-
|
||
{{
|
||
(lldp_dev2_ip6 is defined and (lldp_dev2_ip6 | string | trim | length > 0))
|
||
| ternary(
|
||
'[' ~ (lldp_dev2_ip6 | string | trim) ~ '%' ~ (dev1_iface | string | trim) ~ ']',
|
||
dev2_host | string
|
||
)
|
||
}}
|
||
dev2_tunnel_target_port: "{{ dev2_port | string }}"
|
||
changed_when: false
|
||
|
||
- name: Debug tunnel remote target for DEV2 (IPv4/IPv6)
|
||
when: dev2_conn_method == "tunnel" and debugging | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "dev2_tunnel_target={{ dev2_tunnel_target_host | default(dev2_host) }}:{{ dev2_tunnel_target_port | default(dev2_port) }}"
|
||
|
||
- name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
USER="{{ dev1_user }}"
|
||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
||
sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
|
||
-M -S "{{ _ctrl_sock }}" \
|
||
-L "127.0.0.1:{{ _local_port }}:{{ dev2_tunnel_target_host | default(dev2_host) }}:{{ dev2_tunnel_target_port | default(dev2_port) }}" \
|
||
"${USER}@${HOST}"
|
||
args:
|
||
executable: /bin/bash
|
||
register: start_tunnel
|
||
changed_when: true
|
||
|
||
- name: Probe TCP reachability to DEV2 through the tunnel (nc)
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
nc -z -w5 127.0.0.1 "{{ _local_port }}"
|
||
register: nc_probe
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Stop if tunnel TCP probe failed
|
||
when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
|
||
ansible.builtin.meta: end_host
|
||
|
||
- name: Show passfiles available on controller (ls)
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
|
||
register: dev2_ls
|
||
changed_when: false
|
||
|
||
- name: Debug passfiles presence
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "{{ (dev2_ls.stdout | default('')) | trim }}"
|
||
- "{{ (dev2_ls.stderr | default('')) | trim }}"
|
||
|
||
- name: Try DEV2 login with 'basicpass' (root, tunnel)
|
||
when: dev2_conn_method == "tunnel"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
PORT="{{ _local_port }}"
|
||
sshpass -f basicpass ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_try_basicpass
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Select 'basicpass' if previous login succeeded (tunnel)
|
||
when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_passfile_used: "basicpass"
|
||
dev2_ssh_host: "127.0.0.1"
|
||
dev2_ssh_port: "{{ _local_port }}"
|
||
changed_when: false
|
||
|
||
- name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel)
|
||
when: dev2_conn_method == "tunnel" and dev2_passfile_used == "NONE"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
PORT="{{ _local_port }}"
|
||
sshpass -f basicpass2 ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_try_basicpass2
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Select 'basicpass2' if second login succeeded (tunnel)
|
||
when: dev2_conn_method == "tunnel" and dev2_passfile_used == "NONE" and dev2_try_basicpass2.rc == 0
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_passfile_used: "basicpass2"
|
||
dev2_ssh_host: "127.0.0.1"
|
||
dev2_ssh_port: "{{ _local_port }}"
|
||
changed_when: false
|
||
|
||
# ---------------------------- FALLBACKS (Feb method style) ----------------------------
|
||
- name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed)
|
||
when:
|
||
- dev2_conn_method == "tunnel"
|
||
- dev2_passfile_used == "NONE"
|
||
- (lldp_dev2_ip | default('')) is match('^10\\.')
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ lldp_dev2_ip }}"
|
||
if sshpass -f basicpass ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=20 \
|
||
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
|
||
echo "basicpass"; exit 0
|
||
fi
|
||
if sshpass -f basicpass2 ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=20 \
|
||
"{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
|
||
echo "basicpass2"; exit 0
|
||
fi
|
||
echo "NONE"; exit 1
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_auth_lldp4
|
||
changed_when: false
|
||
failed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Apply LLDP IPv4 fallback decision
|
||
when:
|
||
- dev2_conn_method == "tunnel"
|
||
- dev2_passfile_used == "NONE"
|
||
- (lldp_dev2_ip | default('')) is match('^10\\.')
|
||
- dev2_auth_lldp4 is defined
|
||
- (dev2_auth_lldp4.stdout | trim) in ['basicpass','basicpass2']
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
dev2_conn_method: "lldp4_fallback"
|
||
dev2_passfile_used: "{{ dev2_auth_lldp4.stdout | trim }}"
|
||
dev2_ssh_host: "{{ lldp_dev2_ip }}"
|
||
dev2_ssh_port: 22
|
||
changed_when: false
|
||
|
||
- name: Debug final DEV2 connectivity decision (upgrade will use host/port/passfile below)
|
||
when: debugging | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "dev2_conn_method={{ dev2_conn_method }}"
|
||
- "dev2_passfile_used={{ dev2_passfile_used }}"
|
||
- "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
|
||
- "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
|
||
- "lldp_dev2_ip={{ lldp_dev2_ip | default('') }}"
|
||
- "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('') }}"
|
||
- "dev2_host={{ dev2_host }}:{{ dev2_port }}"
|
||
|
||
# ============================ UPGRADE PROCEDURE BELOW (UNCHANGED) ============================
|
||
|
||
# ---------------------------- Read DEV2 hostname via chosen path ----------------------------
|
||
- name: Read DEV2 hostname via selected connection (busybox-safe)
|
||
when: dev2_passfile_used != "NONE"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_host_read
|
||
changed_when: false
|
||
|
||
- name: Normalize hostnames for strict compare (inventory/DEV1/DEV2)
|
||
ansible.builtin.set_fact:
|
||
_inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
|
||
_dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
|
||
_dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
|
||
|
||
- name: Debug normalized hostnames (JSON-escaped)
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "inv={{ _inv_hn | tojson }}"
|
||
- "dev1={{ _dev1_hn | tojson }}"
|
||
- "dev2={{ _dev2_hn | tojson }}"
|
||
|
||
# -------- New: DEV2 hostname classification (numeric ikejaNNNN vs other) --------
|
||
- name: Detect DEV2 hostname pattern (ikeja<number> or other) and raw form
|
||
ansible.builtin.set_fact:
|
||
_dev2_hn_is_ikeja_numeric: "{{ (_dev2_hn | default('')) is match('^ikeja[0-9]+$') }}"
|
||
_dev2_hn_raw: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') }}"
|
||
changed_when: false
|
||
|
||
- name: Classify DEV2 hostname mismatch type
|
||
ansible.builtin.set_fact:
|
||
_hostname_softfail: "{{ _dev2_hn_is_ikeja_numeric and (_dev2_hn != _inv_hn) }}"
|
||
_hostname_warn: "{{ (not _dev2_hn_is_ikeja_numeric) and (_dev2_hn != _inv_hn) }}"
|
||
changed_when: false
|
||
|
||
- name: Debug DEV2 hostname classification
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "_dev2_hn_is_ikeja_numeric={{ _dev2_hn_is_ikeja_numeric | default(false) }}"
|
||
- "_hostname_softfail={{ _hostname_softfail | default(false) }}"
|
||
- "_hostname_warn={{ _hostname_warn | default(false) }}"
|
||
- "_dev2_hn_raw={{ _dev2_hn_raw | default('') }}"
|
||
|
||
# ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
|
||
- name: Init soft-fail journal flags
|
||
ansible.builtin.set_fact:
|
||
_journal: []
|
||
_prep_blocked: false
|
||
_blocked: false
|
||
|
||
- name: Append connection method decision to journal
|
||
ansible.builtin.set_fact:
|
||
_journal: "{{ _journal + [ 'Connection method for DEV2: ' ~ dev2_conn_method ~ (((dev2_conn_method == 'direct_lldp') | ternary(' (' ~ (dev2_ssh_host | default('')) ~ ')',''))) ] }}"
|
||
|
||
|
||
# ---- Apply hostname policy: soft-fail for other ikejaNNNN, warn+customfield for "weird" names ----
|
||
- name: Soft-block if DEV2 hostname is another ikeja<number> (probable wrong indoor)
|
||
when: _hostname_softfail | default(false)
|
||
ansible.builtin.set_fact:
|
||
_blocked: true
|
||
_journal: "{{ (_journal | default([])) + [ 'DEV2 hostname looks like another ikeja-number device: dev2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ '. Treating as wrong indoor, skipping update.' ] }}"
|
||
|
||
- name: Note non-standard DEV2 hostname and prepare indoor_hostname customfield
|
||
when: _hostname_warn | default(false)
|
||
ansible.builtin.set_fact:
|
||
indoor_hostname_value: "{{ '!' ~ (_dev2_hn_raw | default(_dev2_hn)) }}"
|
||
_journal: "{{ (_journal | default([])) + [ 'DEV2 hostname non-standard: ' ~ _dev2_hn_raw ~ ' (customfield indoor_hostname=' ~ ('!' ~ (_dev2_hn_raw | default(_dev2_hn))) ~ ')' ] }}"
|
||
|
||
- name: Build indoor_hostname customfield payload
|
||
when: _hostname_warn | default(false)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
indoor_hostname_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "customfield_set"
|
||
task_result: "indoor_hostname={{ indoor_hostname_value }}"
|
||
|
||
- name: Publish indoor_hostname custom field to control queue
|
||
delegate_to: localhost
|
||
uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ {
|
||
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
||
'task_name': 'custom_field_set',
|
||
'task_add1': 'indoor_hostname',
|
||
'task_result': indoor_hostname_value
|
||
} | to_json }}"
|
||
payload_encoding: "string"
|
||
changed_when: false
|
||
when: indoor_hostname_value is defined
|
||
|
||
- name: Build specific image marker path on DEV2 (/tmp/prepared_for_<image>)
|
||
ansible.builtin.set_fact:
|
||
_marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}"
|
||
|
||
- name: Count existing preparation markers on DEV2 (best-effort)
|
||
when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"ls /tmp/prepared_for* 2>/dev/null | wc -l"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_prep_count
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Soft-block if preparation markers already present on DEV2
|
||
when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
|
||
|
||
- name: Check bootenv-updated marker on DEV2 (/tmp/bootenv_updated)
|
||
when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"[ -f '/tmp/bootenv_updated' ] && echo PRESENT || echo ABSENT"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_bootenv_marker
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Soft-block if bootenv-updated marker is present on DEV2
|
||
when: dev2_bootenv_marker is defined
|
||
and (dev2_bootenv_marker.stdout | default('') | trim) == 'PRESENT'
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'Bootenv updated marker present on DEV2 (/tmp/bootenv_updated). Skipping image staging/write' ] }}"
|
||
|
||
- name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
|
||
|
||
# ---------------------------- DEV2 version firmux primary check ----------------------------
|
||
- name: Read DEV2 /usr/lib/release/firmux (if present)
|
||
when: dev2_passfile_used != "NONE"
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"cat /usr/lib/release/firmux 2>/dev/null || true"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_firmux
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Debug DEV2 firmux banner (if any)
|
||
when: dev2_firmux is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
|
||
|
||
# ====== INSERTED: two-step normalization preview (rev->r, then space->dash) ======
|
||
- name: "Normalize DEV2 firmux (step 1): replace 'rev ' -> 'r' (debug only)"
|
||
when: dev2_firmux is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
_firmux_step1: "{{ (dev2_firmux.stdout | default('') | trim) | regex_replace('(?i)rev\\s+','r') }}"
|
||
changed_when: false
|
||
|
||
- name: "Debug firmux after step 1 (rev->r)"
|
||
when: _firmux_step1 is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg: "_firmux_step1={{ _firmux_step1 }}"
|
||
|
||
- name: "Normalize DEV2 firmux (step 2): replace space -> '-' (debug only)"
|
||
when: _firmux_step1 is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
_firmux_step2: "{{ _firmux_step1 | replace(' ', '-') }}"
|
||
changed_when: false
|
||
|
||
- name: "Debug firmux after step 2 (space->dash)"
|
||
when: _firmux_step2 is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg: "_firmux_step2={{ _firmux_step2 }}"
|
||
# ====== END INSERTED ======
|
||
|
||
# ===================== Early equality/substring check (same logic as checker) =====================
|
||
- name: Normalize expected target step one compute base string (from image_filename)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
expected_norm_step1: "{{ (image_filename | default('') | trim) }}"
|
||
changed_when: false
|
||
|
||
- name: Extract version core X dot Y dot Z dash rNNNN from image_filename if present
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
expected_norm_core_list: "{{ (image_filename | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
|
||
changed_when: false
|
||
|
||
- name: Choose first extracted core if available
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
|
||
changed_when: false
|
||
|
||
- name: Normalize expected target step two prefer extracted core when available
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
|
||
changed_when: false
|
||
|
||
- name: Capture firmware banner line from DEV2 (raw)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
banner_raw: "{{ (dev2_firmux.stdout | default('') | trim) }}"
|
||
changed_when: false
|
||
|
||
- name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
banner_norm: >-
|
||
{{
|
||
((banner_raw | lower) is search('-r[0-9]+$'))
|
||
| ternary(
|
||
banner_raw,
|
||
(banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1'))
|
||
)
|
||
}}
|
||
changed_when: false
|
||
|
||
- name: Evaluate version match using normalized equality or substring
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
version_match: >-
|
||
{{
|
||
(expected_norm | default('') | length > 0)
|
||
and (
|
||
(banner_norm | default('')) == (expected_norm | default(''))
|
||
or ((banner_norm | default('') | lower) is search((expected_norm | default('') | lower)))
|
||
or ((expected_norm | default('') | lower) is search((banner_norm | default('') | lower)))
|
||
)
|
||
}}
|
||
changed_when: false
|
||
|
||
- name: Debug version compare snapshot (pre-write)
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "expected_norm={{ expected_norm | default('') }}"
|
||
- "banner_raw={{ banner_raw | default('') }}"
|
||
- "banner_norm={{ banner_norm | default('') }}"
|
||
- "force_upgrade={{ force_upgrade | default(false) }}"
|
||
- "version_match={{ version_match | default(false) }}"
|
||
|
||
- name: Soft-block if already on target (skip staging/write unless force-upgrade)
|
||
when:
|
||
- not (force_upgrade | default(false) | bool)
|
||
- version_match | default(false)
|
||
ansible.builtin.set_fact:
|
||
_blocked: true
|
||
_journal: "{{ (_journal | default([])) + [ 'Device already running target image: ' ~ (banner_norm | default('')) ~ ' — skipping write/flip (use force-upgrade=yes to override)' ] }}"
|
||
expected_fw_core_early: "{{ expected_norm | default('') }}"
|
||
current_fw_core: "{{ banner_norm | default('') }}"
|
||
changed_when: false
|
||
|
||
- name: Back-fill compare names for summary (no-op if already set)
|
||
when: not (_blocked | default(false))
|
||
ansible.builtin.set_fact:
|
||
expected_fw_core_early: "{{ expected_fw_core_early | default(expected_norm | default('')) }}"
|
||
current_fw_core: "{{ current_fw_core | default(banner_norm | default('')) }}"
|
||
changed_when: false
|
||
|
||
# ---------------------------- Normalize rebootin early (HOURS) ----------------------------
|
||
- name: Normalize rebootin (strict hours → base seconds/minutes)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
reboot_hours: "{{ (rebootin | default('') | string | trim | int) }}"
|
||
reboot_seconds: "{{ (rebootin | default('') | string | trim | int) * 3600 }}"
|
||
reboot_minutes: "{{ (rebootin | default('') | string | trim | int) * 60 }}"
|
||
reboot_requested: true
|
||
_reboot_requested: true
|
||
|
||
- name: Compute reboot delay (+20s grace) and mirror underscore vars
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
reboot_delay_seconds: "{{ (reboot_seconds | int) + 20 }}"
|
||
reboot_delay_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}"
|
||
_reboot_seconds: "{{ (reboot_seconds | int) + 20 }}"
|
||
_reboot_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}"
|
||
|
||
- name: Debug reboot normalization detail
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "rebootin={{ rebootin | default('UNSET') }}"
|
||
- "reboot_hours={{ reboot_hours }}"
|
||
- "reboot_seconds={{ reboot_seconds }}"
|
||
- "reboot_minutes={{ reboot_minutes }}"
|
||
|
||
- name: Warn if rebootin was not provided by consumer (debug only)
|
||
when: (rebootin | default('') | string | trim) == ''
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg: "WARNING: rebootin is empty or missing. Consumer likely did not pass -e rebootin=<hours>."
|
||
|
||
- name: Debug reboot plan summary
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "reboot_requested={{ _reboot_requested | default(false) }}"
|
||
- "reboot_seconds={{ _reboot_seconds | default(0) }}"
|
||
- "reboot_minutes={{ _reboot_minutes | default(0) }}"
|
||
|
||
- name: Scheduler mode compute action_restart_timestamp
|
||
when: is_run_by_effective == 'scheduler'
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
action_restart_now_ts: "{{ lookup('pipe', 'date +%s') | int }}"
|
||
action_restart_timestamp: "{{ (lookup('pipe', 'date +%s') | int) + (_reboot_seconds | int) }}"
|
||
changed_when: false
|
||
|
||
- name: Build human-readable reboot phrase
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
_reboot_phrase: >-
|
||
{% if not (_reboot_requested | default(false)) -%}
|
||
reboot not requested
|
||
{%- elif (_reboot_seconds | int) <= 20 -%}
|
||
rebooting now
|
||
{%- elif (_reboot_seconds | int) >= 3600 -%}
|
||
reboot scheduled in {{ ((_reboot_seconds | int) // 3600) | int }}h
|
||
{%- else -%}
|
||
reboot scheduled in {{ (_reboot_minutes | int) }}m
|
||
{%- endif %}
|
||
|
||
# ---------------------------- Journal: indoor start (we can proceed) ----------------------------
|
||
- name: Build control queue payload for 'indoor start' journal
|
||
ansible.builtin.set_fact:
|
||
journal_indoor_start:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
Indoor: Dev2 is reachable, starting update.
|
||
conn_method={{ dev2_conn_method }},
|
||
image={{ image_filename }},
|
||
current firmware={{ (dev2_firmux.stdout | default('unknown')) | trim }},
|
||
dev2_hostname={{ _dev2_hn_raw | default(_dev2_hn) }}{% if _hostname_warn | default(false) %} (non-standard; indoor_hostname={{ indoor_hostname_value | default('') }}){% endif %},
|
||
reboot planned in {{ ((_reboot_seconds | int) // 3600) | int }}h
|
||
when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
|
||
- name: Publish 'indoor start' journal to control queue
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_indoor_start | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_journal_indoor_start_resp
|
||
changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
|
||
failed_when: >
|
||
(rmq_journal_indoor_start_resp.status != 200) or
|
||
(rmq_journal_indoor_start_resp.json is not defined) or
|
||
(not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
|
||
when: journal_indoor_start is defined
|
||
delegate_to: localhost
|
||
|
||
# ---------------------------- Stage image on DEV2 (no write yet) ----------------------------
|
||
- name: Check local presence of image file on controller
|
||
delegate_to: localhost
|
||
ansible.builtin.stat:
|
||
path: "{{ image_filename }}"
|
||
register: local_img
|
||
|
||
- name: Soft-block if local image is missing
|
||
when: not local_img.stat.exists
|
||
ansible.builtin.set_fact:
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
|
||
|
||
- name: Compute local md5 of the image (controller)
|
||
when: local_img.stat.exists
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
md5sum "{{ image_filename }}" | awk '{print $1}'
|
||
register: local_md5
|
||
changed_when: false
|
||
|
||
- name: Verify local md5 matches expected
|
||
when: local_img.stat.exists
|
||
delegate_to: localhost
|
||
ansible.builtin.assert:
|
||
that:
|
||
- (local_md5.stdout | trim) == image_md5
|
||
fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
|
||
success_msg: "Local md5 matches expected"
|
||
|
||
- name: Compute local sha256 of the image (if provided)
|
||
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
sha256sum "{{ image_filename }}" | awk '{print $1}'
|
||
register: local_sha256
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Soft-block if local sha256 mismatch/unavailable
|
||
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
|
||
|
||
- name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
|
||
|
||
- name: Read fw_printenv size (line count) on DEV2 (soft health)
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"(fw_printenv 2>/dev/null | wc -l) || echo 0"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_fwenv_wc
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Soft-block if fw_printenv too small (<30 lines)
|
||
when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
|
||
|
||
- name: Check existing DEV2 image md5 (NOFILE if missing)
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_md5_before
|
||
changed_when: false
|
||
|
||
- name: Copy image to DEV2 if missing or md5 mismatch
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" scp \
|
||
-P "$PORT" \
|
||
-O \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
"{{ image_filename }}" "{{ dev2_ssh_user }}@${HOST}:{{ (dev2_ssh_user == 'root') | ternary('/', '') }}{{ dev2_image_dir }}/{{ image_filename }}"
|
||
args:
|
||
executable: /bin/bash
|
||
register: scp_push
|
||
changed_when: true
|
||
|
||
- name: Compute md5 of image on DEV2 after copy (or if already present)
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_md5_after
|
||
changed_when: false
|
||
|
||
- name: Soft-block if DEV2 md5 != expected
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
|
||
|
||
- name: Compute sha256 of image on DEV2 (if provided)
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_sha256_after
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Soft-block if DEV2 sha256 mismatch/unavailable
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
|
||
|
||
- name: Pre-validate image on DEV2 with 'update -c' (non-invasive)
|
||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=30 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"update -c '{{ dev2_image_path }}' 2>&1 || true"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_update_check
|
||
changed_when: false
|
||
|
||
- name: Debug output from 'update -c'
|
||
when: dev2_update_check is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
|
||
|
||
- name: Soft-block if 'update -c' did not return 'valid'
|
||
when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
|
||
ansible.builtin.set_fact:
|
||
_prep_blocked: true
|
||
_blocked: true
|
||
_journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
|
||
|
||
# ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
|
||
- name: Build control queue payload for 'indoor aborted' journal
|
||
ansible.builtin.set_fact:
|
||
journal_indoor_aborted:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
|
||
when: (_blocked | default(false))
|
||
delegate_to: localhost
|
||
|
||
- name: Publish 'indoor aborted' journal to control queue
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_indoor_aborted | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_journal_indoor_aborted_resp
|
||
changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
|
||
failed_when: >
|
||
(rmq_journal_indoor_aborted_resp.status != 200) or
|
||
(rmq_journal_indoor_aborted_resp.json is not defined) or
|
||
(not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
|
||
when: journal_indoor_aborted is defined
|
||
delegate_to: localhost
|
||
|
||
- name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
|
||
when: dev2_conn_method == "tunnel"
|
||
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
|
||
|
||
# ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
|
||
- name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
|
||
when: not (_blocked | default(false))
|
||
block:
|
||
|
||
# --- supervised async write with stall handling ---
|
||
- name: Launch update -w on DEV2 (async)
|
||
delegate_to: localhost
|
||
vars:
|
||
_write_async_cap: 600 # 10 minutes max runtime (tune if needed)
|
||
_write_poll: 20 # poll every 20 seconds
|
||
block:
|
||
- name: Launch update -w on DEV2 (async)
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o AddressFamily=inet \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=0 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"update -w '{{ dev2_image_path }}'"
|
||
args:
|
||
executable: /bin/bash
|
||
async: "{{ _write_async_cap }}"
|
||
poll: "{{ _write_poll }}"
|
||
register: dev2_up_write
|
||
changed_when: true
|
||
failed_when: false
|
||
|
||
- name: Evaluate async result
|
||
ansible.builtin.set_fact:
|
||
_updatew_stalled: >-
|
||
{{ (dev2_up_write.msg is defined) and
|
||
('did not complete' in dev2_up_write.msg | lower) }}
|
||
_updatew_success: >-
|
||
{{
|
||
((dev2_up_write.stdout | default('')) is search('update is complete'))
|
||
or
|
||
((not (dev2_up_write.skipped | default(false))) and ((((dev2_up_write.stdout | default('')) | trim) == '') and (((dev2_up_write.rc | default(1)) | int) == 0)))
|
||
}}
|
||
|
||
- name: Journal stall if update -w timed out
|
||
when: _updatew_stalled | bool
|
||
ansible.builtin.set_fact:
|
||
journal_updatew_stalled:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: "update -w has stalled. Rebooting device; please schedule another upgrade."
|
||
|
||
- name: Publish stall journal to control queue
|
||
when: journal_updatew_stalled is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_updatew_stalled | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_stall_journal_resp
|
||
changed_when: (rmq_stall_journal_resp.json is defined) and (rmq_stall_journal_resp.json.routed | default(false) | bool)
|
||
failed_when: false
|
||
|
||
- name: Schedule immediate reboot (20s delay) after stall
|
||
when: _updatew_stalled | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"nohup /sbin/reboot -d 20 >/dev/null 2>&1 &"
|
||
args:
|
||
executable: /bin/bash
|
||
changed_when: true
|
||
ignore_errors: true
|
||
|
||
- name: Stop play if update -w stalled
|
||
when: _updatew_stalled | bool
|
||
ansible.builtin.meta: end_play
|
||
|
||
- name: Fail play if update -w completed but invalid output
|
||
when: (not _updatew_stalled) and (not _updatew_success)
|
||
ansible.builtin.fail:
|
||
msg: "update -w finished but did not report 'update is complete'"
|
||
|
||
# --- continue normal path if not stalled ---
|
||
- name: Read current active partition on DEV2 (before flip)
|
||
when: not _updatew_stalled | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"fw_printenv active | awk -F= '/^active=/{print \$2}'"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_active_before
|
||
changed_when: false
|
||
failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
|
||
|
||
- name: Determine new active value for DEV2 (flip 1↔2)
|
||
when: not _updatew_stalled | bool and dev2_active_before.stdout is defined
|
||
ansible.builtin.set_fact:
|
||
dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
|
||
|
||
- name: Set fw_setenv active={{ dev2_new_active }} on DEV2
|
||
when: not _updatew_stalled | bool and dev2_new_active is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"fw_setenv active {{ dev2_new_active }}"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_setenv_out
|
||
changed_when: true
|
||
|
||
- name: Verify active partition flipped on DEV2 (read back)
|
||
when: not _updatew_stalled | bool and dev2_new_active is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"fw_printenv active | awk -F= '/^active=/{print \$2}'"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_active_after
|
||
changed_when: false
|
||
failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
|
||
|
||
- name: Create specific prep marker on DEV2 for this image
|
||
when: not _updatew_stalled | bool
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
|
||
"touch '{{ _marker_specific }}'"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_marker_write
|
||
changed_when: true
|
||
ignore_errors: true
|
||
|
||
# ---------------------------- Reboot scheduling (normalized) ----------------------------
|
||
- name: Schedule DEV2 reboot after computed delay (seconds)
|
||
when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
HOST="{{ dev2_ssh_host }}"
|
||
PORT="{{ dev2_ssh_port }}"
|
||
SECS="{{ _reboot_seconds | int }}"
|
||
CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
|
||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
||
-o ConnectTimeout=10 \
|
||
-p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "${CMD}"
|
||
args:
|
||
executable: /bin/bash
|
||
register: dev2_reboot_sched
|
||
changed_when: true
|
||
ignore_errors: true
|
||
|
||
- name: Scheduler mode compute reboot_applied flag (gate for action_restart_timestamp)
|
||
when: is_run_by_effective == 'scheduler'
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
reboot_applied_cf: "{{ (dev2_reboot_sched is defined) and (dev2_reboot_sched.rc is defined) and ((dev2_reboot_sched.rc | int) == 0) }}"
|
||
changed_when: false
|
||
|
||
|
||
- name: Scheduler mode set action_state waiting when reboot initiated
|
||
when: is_run_by_effective == 'scheduler' and (reboot_applied_cf | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ {
|
||
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
||
'task_name': 'custom_field_set',
|
||
'task_add1': 'action_state',
|
||
'task_result': 'waiting'
|
||
} | to_json }}"
|
||
payload_encoding: "string" # <-- ADD THIS
|
||
|
||
- name: Scheduler mode publish action_restart_timestamp custom field to control queue
|
||
when: is_run_by_effective == 'scheduler' and (reboot_applied_cf | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ {
|
||
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
||
'task_name': 'custom_field_set',
|
||
'task_add1': 'action_restart_timestamp',
|
||
'task_result': (action_restart_timestamp | string)
|
||
} | to_json }}"
|
||
payload_encoding: "string"
|
||
changed_when: false
|
||
|
||
|
||
- name: Build 'indoor-restart-scheduled' tag payload
|
||
ansible.builtin.set_fact:
|
||
tag_restart_sched_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "tag_add"
|
||
task_result: "indoor-restart-scheduled"
|
||
when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
|
||
- name: Publish 'indoor-restart-scheduled' tag to control queue
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ tag_restart_sched_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_tag_restart_sched_resp
|
||
changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)
|
||
failed_when: >
|
||
(rmq_tag_restart_sched_resp.status != 200) or
|
||
(rmq_tag_restart_sched_resp.json is not defined) or
|
||
(not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool))
|
||
when: tag_restart_sched_payload is defined
|
||
delegate_to: localhost
|
||
|
||
- name: Note reboot was requested but value is invalid (format warning)
|
||
when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
|
||
ansible.builtin.debug:
|
||
msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"
|
||
|
||
# ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
|
||
- name: Compute write success flag (_write_success)
|
||
ansible.builtin.set_fact:
|
||
_write_success: >-
|
||
{{
|
||
(
|
||
(
|
||
(dev2_up_write.stdout | default('')) is search('update is complete')
|
||
)
|
||
or
|
||
(
|
||
((not (dev2_up_write.skipped | default(false))) and (((dev2_up_write.stdout | default('')) | trim) == '')
|
||
and (((dev2_up_write.rc | default(1)) | int) == 0))
|
||
)
|
||
)
|
||
if (dev2_up_write is defined) else false
|
||
}}
|
||
delegate_to: localhost
|
||
|
||
- name: Debug write result and active banks (quick summary)
|
||
when: not (_blocked | default(false))
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "write_success={{ _write_success | default(false) }}"
|
||
- "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
|
||
- "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
|
||
|
||
- name: Build 'indoor updated' journal payload text
|
||
ansible.builtin.set_fact:
|
||
journal_indoor_updated:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
Indoor: Dev2 prepared for update, {{ _reboot_phrase }}.
|
||
conn_method={{ dev2_conn_method }},
|
||
write_done={{ _write_success }},
|
||
active_before={{ (dev2_active_before.stdout | default('NA')) | trim }},
|
||
active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}
|
||
when: (_write_success | bool)
|
||
delegate_to: localhost
|
||
|
||
- name: Publish 'indoor updated' journal to control queue
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_indoor_updated | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_journal_indoor_updated_resp
|
||
changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
|
||
failed_when: >
|
||
(rmq_journal_indoor_updated_resp.status != 200) or
|
||
(rmq_journal_indoor_updated_resp.json is not defined) or
|
||
(not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
|
||
when: journal_indoor_updated is defined
|
||
delegate_to: localhost
|
||
|
||
# ──────────────────────────────── Derive expected version for the checker (strict) ────────────────────────────────
|
||
- name: Init after-upgrade scheduling vars (attempt=1, delay=reboot + 6m)
|
||
ansible.builtin.set_fact:
|
||
au_attempt: 1
|
||
au_max_attempts: 3
|
||
au_delay_sec: "{{ (reboot_seconds | int) + 360 }}"
|
||
when: not (_blocked | default(false)) and (_write_success | bool)
|
||
delegate_to: localhost
|
||
|
||
- name: Generate correlation ID and UTC timestamp (for after-upgrade tracking)
|
||
ansible.builtin.set_fact:
|
||
au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
|
||
au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
|
||
when: au_delay_sec is defined
|
||
delegate_to: localhost
|
||
|
||
- name: Derive expected target_version from image filename (strict X.Y.Z-rNNNN)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
expected_fw_core: "{{ (image_filename | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default('', true) }}"
|
||
|
||
- name: Debug derived expected target_version for checker
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "image_filename={{ image_filename }}"
|
||
- "expected_fw_core={{ expected_fw_core }}"
|
||
when: expected_fw_core is defined
|
||
|
||
- name: Build after-upgrade check payload (attempt 1)
|
||
ansible.builtin.set_fact:
|
||
afterupgrade_payload:
|
||
task_name: "afterupgrade_indoor_check"
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
target_version_full: "{{ image_filename }}"
|
||
target_version: "{{ expected_fw_core }}"
|
||
attempt: "{{ au_attempt }}"
|
||
max_attempts: "{{ au_max_attempts }}"
|
||
current_delay_sec: "{{ au_delay_sec }}"
|
||
correlation_id: "{{ au_correlation_id }}"
|
||
original_emitted_at: "{{ au_original_emitted_at }}"
|
||
schema_version: 1
|
||
is_run_by: "{{ is_run_by_effective }}"
|
||
when: au_delay_sec is defined
|
||
delegate_to: localhost
|
||
|
||
- name: Debug after-upgrade plan (routing + delay + version)
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "routing_key={{ afterupgrade_routing_key }}"
|
||
- "x-delay(ms)={{ (au_delay_sec | int) * 1000 }}"
|
||
- "target_version={{ afterupgrade_payload.target_version | default('NA') }}"
|
||
when: afterupgrade_payload is defined
|
||
delegate_to: localhost
|
||
|
||
- name: Debug after-upgrade payload JSON (exactly what will be sent)
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg: "{{ afterupgrade_payload | to_json }}"
|
||
when: afterupgrade_payload is defined
|
||
|
||
- name: Publish delayed after-upgrade check (headers.x-delay) to holding exchange
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
headers:
|
||
x-delay: "{{ (au_delay_sec | int) * 1000 }}"
|
||
routing_key: "{{ afterupgrade_routing_key }}"
|
||
payload: "{{ afterupgrade_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_afterupgrade_resp
|
||
changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
|
||
failed_when: >
|
||
(rmq_afterupgrade_resp.status != 200) or
|
||
(rmq_afterupgrade_resp.json is not defined)
|
||
when: afterupgrade_payload is defined
|
||
delegate_to: localhost
|
||
|
||
# ---------------------------- Final operator summary (one-liners) ----------------------------
|
||
- name: Summary key outcomes (one-liners)
|
||
delegate_to: localhost
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "dev2_conn_method={{ dev2_conn_method }}"
|
||
- "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
|
||
- "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
|
||
- "dev2_passfile_used={{ dev2_passfile_used }}"
|
||
- "dev2_hostname_raw={{ _dev2_hn_raw | default('') }}"
|
||
- "hostname_softfail={{ _hostname_softfail | default(false) }}"
|
||
- "hostname_warn={{ _hostname_warn | default(false) }}"
|
||
- "indoor_hostname_value={{ indoor_hostname_value | default('') }}"
|
||
- "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
|
||
- "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
|
||
- "current_fw_core={{ current_fw_core | default('') }}"
|
||
- "force_upgrade={{ force_upgrade | default(false) }}"
|
||
- "local_image_present={{ local_img.stat.exists | default(false) }}"
|
||
- "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
|
||
- "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
|
||
- "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
|
||
- "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}"
|
||
- "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
|
||
- "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
|
||
- "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
|
||
- "reboot_requested={{ reboot_requested | default(false) }}"
|
||
- "reboot_delay_seconds={{ reboot_delay_seconds if (reboot_requested | default(false)) else 'NA' }}"
|
||
- "reboot_delay_minutes={{ reboot_delay_minutes if (reboot_requested | default(false)) else 'NA' }}"
|
||
- "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
|
||
- "prep_blocked={{ _prep_blocked | default(false) }}"
|
||
- "blocked={{ _blocked | default(false) }}"
|
||
- "journal={{ (_journal | default([])) | join(' || ') }}"
|
||
|
||
post_tasks:
|
||
- name: Scheduler mode report failure to NetBox when update needed but reboot not applied
|
||
when: is_run_by_effective == 'scheduler' and (((not (version_match | default(false))) or (force_upgrade | default(false)))) and (not ((dev2_reboot_sched is defined) and (dev2_reboot_sched.rc is defined) and ((dev2_reboot_sched.rc | int) == 0)))
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ {
|
||
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
||
'task_name': 'custom_field_set',
|
||
'task_add1': 'action_state',
|
||
'task_result': 'failed'
|
||
} | to_json }}"
|
||
payload_encoding: "string"
|
||
changed_when: false
|
||
|
||
- name: Cleanup (always)
|
||
block:
|
||
- ansible.builtin.debug:
|
||
msg: "Entering cleanup block"
|
||
changed_when: false
|
||
delegate_to: localhost
|
||
always:
|
||
- name: Close SSH ControlMaster (best-effort)
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Remove tunnel control dir (best-effort)
|
||
delegate_to: localhost
|
||
ansible.builtin.file:
|
||
path: "{{ _ctrl_dir | default('/tmp/none') }}"
|
||
state: absent
|
||
ignore_errors: true
|
||
|
||
- name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address' and 'Address not available')
|
||
when: dev2_conn_method | default('') == "tunnel"
|
||
ansible.builtin.raw: >
|
||
{{ pathprefix }}
|
||
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
|
||
register: del_ip
|
||
changed_when: del_ip.rc == 0
|
||
failed_when: >
|
||
del_ip.rc != 0
|
||
and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
|
||
and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
|
||
and ('Address not available' not in (del_ip.stdout | default('')))
|
||
and ('Address not available' not in (del_ip.stderr | default('')))
|
||
and ('RTNETLINK answers: Address not available' not in (del_ip.stdout | default('')))
|
||
and ('RTNETLINK answers: Address not available' not in (del_ip.stderr | default('')))
|
||
|
||
|
||
- name: Debug temp IP removal result
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "del_ip.rc={{ del_ip.rc | default('') }}"
|
||
- "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
|
||
- "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
|
||
when: del_ip is defined |