From 05d02e1e899950d0b5db2770cc2c287ae48d4bbd Mon Sep 17 00:00:00 2001 From: pavel Date: Mon, 17 Nov 2025 16:09:31 +0200 Subject: [PATCH] 1609 --- .../update-indoor-bootenv.yml | 1460 +++++++++++++++++ 1 file changed, 1460 insertions(+) create mode 100644 files/ansible-playbooks/update-indoor-bootenv.yml diff --git a/files/ansible-playbooks/update-indoor-bootenv.yml b/files/ansible-playbooks/update-indoor-bootenv.yml new file mode 100644 index 0000000..fd7398d --- /dev/null +++ b/files/ansible-playbooks/update-indoor-bootenv.yml @@ -0,0 +1,1460 @@ +- name: Second-line indoor bootenv update via DEV1 → LLDP/tunnel → DEV2 (non-invasive control path) + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30" + + # Bootenv image to stage on DEV2 + bootenv_filename: "fox200_bootenv.bin" + bootenv_sha256: "ea928431c905a6a3f4c418be79e66bbd4cc6894a81e94b2e519d3e0e0aeba63c" + + # Aliases to reuse existing image_* logic (no firmware anymore, just bootenv) + image_filename: "{{ bootenv_filename }}" + image_md5: "" + image_sha256: "{{ bootenv_sha256 }}" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Bootenv marker path on DEV2 + bootenv_marker: "/tmp/bootenv_updated" + + # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + # ---------------- New: debugging toggle for newly added debug tasks ---------------- + debugging: true + + # ---------------- New: abstracted DEV2 connection (decided early) ---------------- + # "tunnel" (default) or "direct_lldp" + dev2_conn_method: "tunnel" + dev2_ssh_host: "" + dev2_ssh_port: "" + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname (busybox-safe) + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Debug incoming parameters from consumer and defaults + delegate_to: localhost + ansible.builtin.debug: + msg: + - "rebootin={{ rebootin | default('UNSET') }}" + - "bootenv_filename={{ bootenv_filename | default('UNSET') }}" + - "bootenv_sha256={{ bootenv_sha256 | default('UNSET') }}" + - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}" + - "inventory_hostname={{ inventory_hostname }}" + + - name: Stop early if connected DEV1 hostname != inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ============================ LLDP-FIRST CONNECTION DECISION ============================ + - name: Compute hostname digits key for LLDP lookup (DEV2) + ansible.builtin.set_fact: + dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}" + changed_when: false + + - name: Discover DEV2 candidate IP via LLDP on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + DIGITS="{{ dev2_lldp_digits }}"; + cat /var/run/lldp_server.json 2>/dev/null \ + | grep "${DIGITS}" -A 10 \ + | grep address \ + | grep -vE 'subtype|ipv6' \ + | awk -F'"' '{ print $4 }' \ + | head -n1 + register: dev2_lldp_ip_raw + changed_when: false + failed_when: false + + - name: Capture LLDP-derived DEV2 IP (if any) + ansible.builtin.set_fact: + lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}" + changed_when: false + + - name: Classify LLDP candidate range + delegate_to: localhost + ansible.builtin.set_fact: + lldp_ip_class: >- + {% set ip = (lldp_dev2_ip | default('')) %} + {% if ip == '' %}none + {% elif ip.startswith('10.') %}10 + {% elif ip.startswith('192.168.') %}192_168 + {% else %}other{% endif %} + changed_when: false + + - name: Debug LLDP candidate and classification (new debug) + when: debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "LLDP digits={{ dev2_lldp_digits | default('') }}" + - "LLDP candidate IP={{ lldp_dev2_ip | default('') }}" + - "LLDP class={{ lldp_ip_class | default('none') }}" + + # Decide dev2_conn_method = direct_lldp for 10.x, tunnel otherwise + - name: Set connection method to tunnel by default + delegate_to: localhost + ansible.builtin.set_fact: + dev2_conn_method: "tunnel" + changed_when: false + + - name: Switch to direct LLDP mode for 10.x.x.x + when: lldp_ip_class == "10" + delegate_to: localhost + ansible.builtin.set_fact: + dev2_conn_method: "direct_lldp" + changed_when: false + + - name: Debug connection method decision (new debug) + when: debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_conn_method={{ dev2_conn_method }}" + - "lldp_dev2_ip={{ lldp_dev2_ip | default('') }}" + + # Direct LLDP 10.x login (no tunnel, reusing two-step auth style) + - name: Try DEV2 login via direct LLDP IP with 'basicpass' (10.x) + when: dev2_conn_method == "direct_lldp" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ lldp_dev2_ip }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_lldp_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select 'basicpass' for direct LLDP if previous login succeeded + when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login via direct LLDP IP with 'basicpass2' (10.x, only if first failed) + when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used is not defined) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ lldp_dev2_ip }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_lldp_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select 'basicpass2' for direct LLDP if second login succeeded + when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined and dev2_lldp_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Mark DEV2 auth as NONE for direct LLDP if both attempts failed + when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug direct LLDP auth decision (new debug) + when: dev2_conn_method == "direct_lldp" and debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_lldp_try_basicpass.rc={{ dev2_lldp_try_basicpass.rc | default('NA') }}" + - "dev2_lldp_try_basicpass2.rc={{ dev2_lldp_try_basicpass2.rc | default('NA') }}" + - "dev2_passfile_used={{ dev2_passfile_used | default('NONE') }}" + - "lldp_dev2_ip={{ lldp_dev2_ip | default('') }}" + + # Set host/port for direct LLDP if auth succeeded + - name: Set direct LLDP DEV2 SSH host/port + when: dev2_conn_method == "direct_lldp" and dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.set_fact: + dev2_ssh_host: "{{ lldp_dev2_ip }}" + dev2_ssh_port: 22 + changed_when: false + + # ---------------------------- Idempotent temp IP on DEV1 (TUNNEL ONLY) ---------------------------- + - name: Add temporary IP on DEV1 (tolerate 'File exists') + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug result of adding temp IP to DEV1 + when: dev2_conn_method == "tunnel" + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 (TUNNEL) ---------------------------- + - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Capture discovered DEV2 MAC (if any) + when: dev2_conn_method == "tunnel" + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug ARP action summary on DEV1 + when: dev2_conn_method == "tunnel" + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Note skipping static ARP add (no MAC discovered) + when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0) + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation (TUNNEL ONLY) ---------------------------- + - name: Pick a free local TCP port for the tunnel (controller side) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + # (moved up) Stop immediately if no free local port was found + - name: Stop if no free local port was found + when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0 + ansible.builtin.meta: end_host + + # (moved up) Set chosen port and control socket path + - name: Record chosen local port and create control dir for SSH ControlMaster + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build path for SSH ControlMaster socket + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug picked local port (controller) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "picked_local_port={{ _local_port }}" + - "ctrl_sock={{ _ctrl_sock }}" + - "dev1_host={{ ansible_host | default(inventory_hostname) }}" + - "dev2_target={{ dev2_host }}:{{ dev2_port }}" + + - name: Show current listeners on picked port (ss/lsof) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + P="{{ _local_port }}" + { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p' + { lsof -nP -iTCP:"{{ _local_port }}" -sTCP:LISTEN 2>/dev/null || true; } + register: port_listeners_before + changed_when: false + failed_when: false + + - name: Debug listeners on picked port (before starting tunnel) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "listeners_before:\n{{ (port_listeners_before.stdout | default('')) | trim }}" + + - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 (TUNNEL ONLY) ---------------------------- + - name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1 + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Debug ControlMaster start result (rc/stdout/stderr) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "start_tunnel.rc={{ start_tunnel.rc | default('NA') }}" + - "start_tunnel.stdout={{ (start_tunnel.stdout | default('')) | trim }}" + - "start_tunnel.stderr={{ (start_tunnel.stderr | default('')) | trim }}" + + - name: Show who is listening now on the local port (post-start) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + P="{{ _local_port }}" + echo "== ss -ltnp on :${P} ==" + { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}' + echo "== lsof LISTEN on :${P} ==" + { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; } + register: port_listeners_after + changed_when: false + failed_when: false + + - name: Debug listeners on picked port (after starting tunnel) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (port_listeners_after.stdout | default('')) | trim }}" + + - name: Wait a moment for tunnel to settle + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify ControlMaster is running (ssh -O check) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug ControlMaster status + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + - name: Debug ControlMaster check (full rc/stdout/stderr) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tun_check.rc={{ tun_check.rc | default('NA') }}" + - "tun_check.stdout={{ (tun_check.stdout | default('')) | trim }}" + - "tun_check.stderr={{ (tun_check.stderr | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth (TUNNEL) ---------------------------- + - name: Probe TCP reachability to DEV2 through the tunnel (nc) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug tunnel reachability result + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP probe failed + when: dev2_conn_method == "tunnel" and nc_probe.rc != 0 + ansible.builtin.meta: end_host + + - name: Show passfiles available on controller (ls) + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles presence + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + - name: Refresh ARP 2 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Single banner probe (kept, TUNNEL ONLY) ---------------------------- + - name: Probe SSH banner through tunnel (pre-auth, quick) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + ssh -p "$PORT" \ + -o PreferredAuthentications=none \ + -o PubkeyAuthentication=no \ + -o KbdInteractiveAuthentication=no \ + -o PasswordAuthentication=no \ + -o NumberOfPasswordPrompts=0 \ + -o ConnectTimeout=5 \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -vvv root@127.0.0.1 true 2>&1 || true + register: tunnel_banner_probe + changed_when: false + failed_when: false + + - name: Debug SSH preauth probe (first 40 lines) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (tunnel_banner_probe.stdout | default('') | split('\n'))[:40] | join('\n') }}" + + # ---------------------------- Pick DEV2 password for root (TUNNEL) ---------------------------- + - name: Try DEV2 login with 'basicpass' (root, tunnel) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Snapshot listeners on local tunnel port (after basicpass try) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + P="{{ _local_port }}" + echo "== ss -ltnp on :${P} ==" + { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}' + echo "== lsof LISTEN on :${P} ==" + { lsof -nP -iTCP:"{{ _local_port }}" -sTCP:LISTEN 2>/dev/null || true; } + echo "== ps/grep ControlMaster by ControlPath ==" + ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true + register: listeners_after_basicpass + changed_when: false + failed_when: false + + - name: Debug auth try context (basicpass) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "auth_try=basicpass rc={{ dev2_try_basicpass.rc | default('NA') }}" + - "local_port={{ _local_port }}" + - "ctrl_sock={{ _ctrl_sock }}" + - "listeners:\n{{ (listeners_after_basicpass.stdout | default('')) | trim }}" + + - name: Select 'basicpass' if previous login succeeded (tunnel) + when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel) + when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Snapshot listeners on local tunnel port (after basicpass2 try) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + P="{{ _local_port }}" + echo "== ss -ltnp on :${P} ==" + { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}' + echo "== lsof LISTEN on :${P} ==" + { lsof -nP -iTCP:"{{ _local_port }}" -sTCP:LISTEN 2>/dev/null || true; } + echo "== ps/grep ControlMaster by ControlPath ==" + ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true + register: listeners_after_basicpass2 + changed_when: false + failed_when: false + + - name: Debug auth try context (basicpass2) + when: dev2_conn_method == "tunnel" and dev2_try_basicpass2 is defined + delegate_to: localhost + ansible.builtin.debug: + msg: + - "auth_try=basicpass2 rc={{ dev2_try_basicpass2.rc | default('NA') }}" + - "local_port={{ _local_port }}" + - "ctrl_sock={{ _ctrl_sock }}" + - "listeners:\n{{ (listeners_after_basicpass2.stdout | default('')) | trim }}" + + - name: Select 'basicpass2' if second login succeeded (tunnel) + when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Mark DEV2 auth as NONE if both attempts failed (tunnel) + when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + # ---------------------------- Set abstracted host/port for tunnel mode ---------------------------- + - name: Set DEV2 SSH host/port for tunnel mode + when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.set_fact: + dev2_ssh_host: "127.0.0.1" + dev2_ssh_port: "{{ _local_port }}" + changed_when: false + + # ---------------------------- Read DEV2 hostname via chosen path ---------------------------- + - name: Read DEV2 hostname via selected connection (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare (inventory/DEV1/DEV2) + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug normalized hostnames (JSON-escaped) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Guard DEV2 hostname must equal inventory AND DEV1 (prevents IP churn mistakes) + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch info to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname check) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal (hostname mismatch) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Init soft-fail journal flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Append connection method decision to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Connection method for DEV2: ' ~ dev2_conn_method ~ (dev2_conn_method == 'direct_lldp' | ternary(' (' ~ (dev2_ssh_host | default('')) ~ ')','')) ] }}" + + - name: Build bootenv marker path on DEV2 + ansible.builtin.set_fact: + _marker_specific: "{{ bootenv_marker }}" + + - name: Count existing preparation markers on DEV2 (best-effort) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if preparation markers already present on DEV2 + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping bootenv update' ] }}" + + - name: Check if bootenv marker already present on DEV2 + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "[ -f '{{ _marker_specific }}' ] && echo PRESENT || echo ABSENT" + args: + executable: /bin/bash + register: dev2_bootenv_marker_state + changed_when: false + ignore_errors: true + + - name: Soft-block if bootenv marker already present + when: dev2_bootenv_marker_state is defined and (dev2_bootenv_marker_state.stdout | default('') | trim) == 'PRESENT' + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Bootenv marker already present on DEV2; skipping bootenv update' ] }}" + + - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 /usr/lib/release/firmux (if present) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug DEV2 firmux banner (if any) + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- + - name: Normalize rebootin (strict hours → base seconds/minutes) + delegate_to: localhost + ansible.builtin.set_fact: + reboot_hours: "{{ (rebootin | default('') | string | trim | int) }}" + reboot_seconds: "{{ (rebootin | default('') | string | trim | int) * 3600 }}" + reboot_minutes: "{{ (rebootin | default('') | string | trim | int) * 60 }}" + reboot_requested: true + _reboot_requested: true + + - name: Compute reboot delay (+20s grace) and mirror underscore vars + delegate_to: localhost + ansible.builtin.set_fact: + reboot_delay_seconds: "{{ (reboot_seconds | int) + 20 }}" + reboot_delay_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}" + _reboot_seconds: "{{ (reboot_seconds | int) + 20 }}" + _reboot_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}" + + - name: Debug reboot normalization detail + delegate_to: localhost + ansible.builtin.debug: + msg: + - "rebootin={{ rebootin | default('UNSET') }}" + - "reboot_hours={{ reboot_hours }}" + - "reboot_seconds={{ reboot_seconds }}" + - "reboot_minutes={{ reboot_minutes }}" + + - name: Warn if rebootin was not provided by consumer (debug only) + when: (rebootin | default('') | string | trim) == '' + delegate_to: localhost + ansible.builtin.debug: + msg: "WARNING: rebootin is empty or missing. Consumer likely did not pass -e rebootin=." + + - name: Debug reboot plan summary + delegate_to: localhost + ansible.builtin.debug: + msg: + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_seconds={{ _reboot_seconds | default(0) }}" + - "reboot_minutes={{ _reboot_minutes | default(0) }}" + + - name: Build human-readable reboot phrase + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_phrase: >- + {% if not (_reboot_requested | default(false)) -%} + reboot not requested + {%- elif (_reboot_seconds | int) <= 20 -%} + rebooting now + {%- elif (_reboot_seconds | int) >= 3600 -%} + reboot scheduled in {{ ((_reboot_seconds | int) // 3600) | int }}h + {%- else -%} + reboot scheduled in {{ (_reboot_minutes | int) }}m + {%- endif %} + + # ---------------------------- Journal: indoor start (we can proceed) ---------------------------- + - name: Build control queue payload for 'indoor start' journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting bootenv update. + conn_method={{ dev2_conn_method }}, + bootenv={{ bootenv_filename }}, + current firmware={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot planned in {{ ((_reboot_seconds | int) // 3600) | int }}h + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish 'indoor start' journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage bootenv on DEV2 (no write yet) ---------------------------- + - name: Check local presence of bootenv file on controller + delegate_to: localhost + ansible.builtin.stat: + path: "{{ bootenv_filename }}" + register: local_img + + - name: Soft-block if local bootenv image is missing + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local bootenv image missing on controller: ' ~ bootenv_filename ] }}" + + - name: Compute local sha256 of the bootenv image (controller) + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ bootenv_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local bootenv sha256 mismatch/unavailable + when: local_img.stat.exists and (bootenv_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (bootenv_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local bootenv sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (bootenv_sha256 | trim) ] }}" + + - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # fw_printenv health before bootenv update (inverted logic vs firmware play) + - name: Read fw_printenv size (line count) on DEV2 (soft health) + when: dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv already looks healthy (>=30 lines) + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) >= 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv already has ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (>=30). Skipping bootenv update' ] }}" + + # ---------------------------- Bootenv staging on DEV2 ---------------------------- + - name: Check existing bootenv sha256 on DEV2 (NOSHA if missing) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "[ -f '{{ dev2_image_path }}' ] && sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_before + changed_when: false + + - name: Copy bootenv image to DEV2 if missing or sha256 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_sha256_before.stdout | trim) != (bootenv_sha256 | trim)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ bootenv_filename }}" "{{ dev2_ssh_user }}@${HOST}:{{ dev2_ssh_user == 'root' | ternary('/', '') }}{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_bootenv + changed_when: true + + - name: Compute sha256 of bootenv image on DEV2 after copy (or if already present) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 bootenv sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (bootenv_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (bootenv_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote bootenv sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (bootenv_sha256 | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for 'indoor aborted' journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish 'indoor aborted' journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ============================ ACTUAL BOOTENV WRITE (only if not blocked) ============================ + - name: Upgrade bootenv on DEV2 (guarded by soft-block) + when: not (_blocked | default(false)) + block: + + - name: Write bootenv to /dev/mtdblock1 on DEV2 (dd) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "dd if='{{ dev2_image_path }}' of=/dev/mtdblock1" + args: + executable: /bin/bash + register: dev2_bootenv_dd + changed_when: true + failed_when: dev2_bootenv_dd.rc != 0 + + - name: Debug dd output (bootenv) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dd.rc={{ dev2_bootenv_dd.rc | default('NA') }}" + - "dd.stdout={{ (dev2_bootenv_dd.stdout | default('')) | trim }}" + - "dd.stderr={{ (dev2_bootenv_dd.stderr | default('')) | trim }}" + + - name: Run sync on DEV2 after bootenv write + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "sync" + args: + executable: /bin/bash + register: dev2_bootenv_sync + changed_when: true + failed_when: dev2_bootenv_sync.rc != 0 + + - name: Debug sync output (bootenv) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "sync.rc={{ dev2_bootenv_sync.rc | default('NA') }}" + - "sync.stdout={{ (dev2_bootenv_sync.stdout | default('')) | trim }}" + - "sync.stderr={{ (dev2_bootenv_sync.stderr | default('')) | trim }}" + + - name: Set bootenv write success flag + delegate_to: localhost + ansible.builtin.set_fact: + _bootenv_write_success: "{{ (dev2_bootenv_dd.rc | default(1)) == 0 and (dev2_bootenv_sync.rc | default(1)) == 0 }}" + + - name: Create bootenv marker on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule DEV2 reboot after computed delay (seconds) + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build 'indoor-restart-scheduled' tag payload + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish 'indoor-restart-scheduled' tag to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + - name: Note reboot was requested but value is invalid (format warning) + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Compute write success flag (_write_success) + ansible.builtin.set_fact: + _write_success: "{{ _bootenv_write_success | default(false) }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Debug write result (bootenv) + when: not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "bootenv_write_success={{ _write_success | default(false) }}" + + - name: Build 'indoor updated' journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 bootenv updated, {{ _reboot_phrase }}. + conn_method={{ dev2_conn_method }}, + write_done={{ _write_success }}, + active_before=NA, + active_after=NA + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish 'indoor updated' journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + # ──────────────────────────────── After-upgrade scheduling (same mechanism, bootenv-focused) ──────────────────────────────── + - name: Init after-upgrade scheduling vars (attempt=1, delay=reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ (reboot_seconds | int) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and UTC timestamp (for after-upgrade tracking) + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Derive expected target_version from bootenv context (placeholder) + delegate_to: localhost + ansible.builtin.set_fact: + expected_fw_core: "" + when: au_delay_sec is defined + + - name: Debug derived expected target_version for checker + delegate_to: localhost + ansible.builtin.debug: + msg: + - "bootenv_filename={{ bootenv_filename }}" + - "expected_fw_core={{ expected_fw_core }}" + when: expected_fw_core is defined + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version_full: "{{ bootenv_filename }}" + target_version: "{{ expected_fw_core }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug after-upgrade plan (routing + delay + version) + ansible.builtin.debug: + msg: + - "routing_key={{ afterupgrade_routing_key }}" + - "x-delay(ms)={{ (au_delay_sec | int) * 1000 }}" + - "target_version={{ afterupgrade_payload.target_version | default('NA') }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Debug after-upgrade payload JSON (exactly what will be sent) + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ afterupgrade_payload | to_json }}" + when: afterupgrade_payload is defined + + - name: Publish delayed after-upgrade check (headers.x-delay) to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + # ---------------------------- Final operator summary (one-liners) ---------------------------- + - name: Summary key outcomes (one-liners) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_conn_method={{ dev2_conn_method }}" + - "dev2_ssh_host={{ dev2_ssh_host | default('') }}" + - "dev2_ssh_port={{ dev2_ssh_port | default('') }}" + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "bootenv_filename={{ bootenv_filename }}" + - "bootenv_sha256={{ bootenv_sha256 }}" + - "local_bootenv_present={{ local_img.stat.exists | default(false) }}" + - "local_sha256={{ (local_sha256.stdout | default('NA')) | trim if (local_sha256 is defined) else 'NA' }}" + - "dev2_sha256_before={{ (dev2_sha256_before.stdout | default('NA')) | trim if (dev2_sha256_before is defined) else 'NA' }}" + - "dev2_sha256_after={{ (dev2_sha256_after.stdout | default('NA')) | trim if (dev2_sha256_after is defined) else 'NA' }}" + - "bootenv_write_success={{ _bootenv_write_success | default(false) }}" + - "reboot_requested={{ reboot_requested | default(false) }}" + - "reboot_delay_seconds={{ reboot_delay_seconds if (reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_minutes={{ reboot_delay_minutes if (reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup (always) + block: + - ansible.builtin.debug: + msg: "Entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close SSH ControlMaster (best-effort) + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove tunnel control dir (best-effort) + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address') + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug temp IP removal result + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined