diff --git a/files/ansible-playbooks/update-indoor-224r6820-scheduler.yml b/files/ansible-playbooks/update-indoor-224r6820-scheduler.yml new file mode 100644 index 0000000..e603bd5 --- /dev/null +++ b/files/ansible-playbooks/update-indoor-224r6820-scheduler.yml @@ -0,0 +1,97 @@ +# update-indoor-scheduler.yml +# Single nbplay invocation: +# Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode) +# Phase 2: Refresh ansible_host from nb_ip, reset_connection +# Phase 3: Run update-indoor6-stable.yml + + +- hosts: all + gather_facts: no + vars: + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + tasks: [] + +- import_playbook: sot-updater-iponly.yml + vars: + sot_wrapper_mode: true + + +- name: Phase 2 | Refresh in-memory target IP after iponly + hosts: all + gather_facts: no + vars: + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + tasks: + - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout" + vars: + _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}" + _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}" + _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}" + _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}" + set_fact: + nb_ip: "{{ _ip_raw }}" + nb_ip_ok: "{{ (_ip_raw | length) > 0 }}" + nb_ip_err: "{{ _fail_line }}" + changed_when: false + + - name: "Wrapper | Abort if nb_ip not available" + fail: + msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}" + when: not nb_ip_ok + + - name: "Wrapper | Debug before ansible_host override" + debug: + msg: + - "inventory_hostname={{ inventory_hostname }}" + - "ansible_host(before)={{ ansible_host | default('UNSET') }}" + - "ansible_port={{ ansible_port | default('22') }}" + - "nb_ip={{ nb_ip | default('UNSET') }}" + changed_when: false + + - name: "Wrapper | Update ansible_host to nb_ip" + set_fact: + ansible_host: "{{ nb_ip }}" + changed_when: false + + - name: "Wrapper | Debug after ansible_host override" + debug: + msg: + - "inventory_hostname={{ inventory_hostname }}" + - "ansible_host(after)={{ ansible_host | default('UNSET') }}" + - "ansible_port={{ ansible_port | default('22') }}" + changed_when: false + + - name: "Wrapper | Reset connection after ansible_host change" + meta: reset_connection + + - name: "Wrapper | Debug after reset_connection" + debug: + msg: + - "inventory_hostname={{ inventory_hostname }}" + - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}" + - "ansible_port={{ ansible_port | default('22') }}" + changed_when: false + + + - name: "Wrapper | Pause 2s before next playbook" + pause: + seconds: 2 + changed_when: false + + +- import_playbook: update-indoor222-r6820.yml + vars: + is_run_by: "scheduler" diff --git a/files/ansible-playbooks/update-indoor222-r6820.yml b/files/ansible-playbooks/update-indoor222-r6820.yml new file mode 100644 index 0000000..509ca38 --- /dev/null +++ b/files/ansible-playbooks/update-indoor222-r6820.yml @@ -0,0 +1,1810 @@ +# update-indoor6-stable.yml (Indoor5-stable + surgical update-w success/journaling fixes) +- name: Second-line indoor upgrade via DEV1 → LLDP/tunnel → DEV2 (non-invasive control path) + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler) + is_run_by_effective: "{{ is_run_by | default('manual') }}" + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # Connectivity module knobs (Feb method) + arping_iface: "eth0" + ssh_timeout: 30 + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30" + + # Image to stage on DEV2 (we validate first; the actual write happens later) + image_filename: "2.2.2-r6820.bin" + image_md5: "d6f55a0807a6ce6270759272b3ea8b2f" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + # ---------------- New: debugging toggle for newly added debug tasks ---------------- + debugging: true + + # ---------------- New: abstracted DEV2 connection (decided early) ---------------- + # "tunnel" (default) or "direct_lldp" or "lldp4_fallback" + dev2_conn_method: "tunnel" + dev2_ssh_host: "" + dev2_ssh_port: "" + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname (busybox-safe) + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Debug incoming parameters from consumer and defaults + delegate_to: localhost + ansible.builtin.debug: + msg: + - "rebootin={{ rebootin | default('UNSET') }}" + - "image_filename={{ image_filename | default('UNSET') }}" + - "image_md5={{ image_md5 | default('UNSET') }}" + - "image_sha256={{ image_sha256 | default('UNSET') }}" + - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}" + - "inventory_hostname={{ inventory_hostname }}" + - "is_run_by={{ is_run_by_effective }}" + + # ====== NEW: pick up force-upgrade flag from CLI and TEMP override ====== + - name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes) + ansible.builtin.set_fact: + force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | trim }}" + + # >>> TEMPORARY OVERRIDE (REMOVE THIS TASK LATER) <<< + - name: TEMPORARILY set force-upgrade to yes (REMOVE BEFORE COMMITTING) + ansible.builtin.set_fact: + force_upgrade_raw: "no" + # <<< END TEMPORARY >>> + + - name: Normalize force-upgrade flag to boolean + ansible.builtin.set_fact: + force_upgrade: "{{ (force_upgrade_raw | string | trim) | bool }}" + + - name: Debug force-upgrade effective + delegate_to: localhost + ansible.builtin.debug: + msg: + - "force_upgrade_raw={{ force_upgrade_raw | default('') }}" + - "force_upgrade={{ force_upgrade | default(false) }}" + + - name: Stop early if connected DEV1 hostname != inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ============================ CONNECTIVITY MODULE (FEB "BEST-KNOWN" METHOD) ============================ + + - name: Compute hostname digits key for LLDP lookup (DEV2) + ansible.builtin.set_fact: + dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}" + changed_when: false + + - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + DIGITS="{{ dev2_lldp_digits }}"; + cat /var/run/lldp_server.json 2>/dev/null \ + | grep "${DIGITS}" -A 10 \ + | grep address \ + | grep -vE 'subtype|ipv6' \ + | awk -F'"' '{ print $4 }' \ + | head -n1 + register: dev2_lldp_ip_raw + changed_when: false + failed_when: false + + - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + DIGITS="{{ dev2_lldp_digits }}"; + cat /var/run/lldp_server.json 2>/dev/null \ + | grep "${DIGITS}" -A 15 \ + | grep 'address_ipv6' \ + | awk -F'"' '{ print $4 }' \ + | head -n1 + register: dev2_lldp_ip6_raw + changed_when: false + failed_when: false + + - name: Capture LLDP-derived DEV2 IP facts + ansible.builtin.set_fact: + lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}" + lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}" + changed_when: false + + - name: Classify LLDP IPv4 candidate + delegate_to: localhost + ansible.builtin.set_fact: + lldp_ip_class: >- + {% set ip = (lldp_dev2_ip | default('')) %} + {% if ip == '' %}none + {% elif ip.startswith('10.') %}10 + {% elif ip.startswith('192.168.') %}192_168 + {% else %}other{% endif %} + changed_when: false + + - name: Debug LLDP candidates + when: debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "LLDP digits={{ dev2_lldp_digits | default('') }}" + - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('') }}" + - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}" + - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('') }}" + + - name: Override dev2_host from LLDP when candidate is 192.168.x.x + when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0) + delegate_to: localhost + ansible.builtin.set_fact: + dev2_host: "{{ lldp_dev2_ip | trim }}" + changed_when: false + + - name: Set connection method initial (direct_lldp if 10.x else tunnel) + delegate_to: localhost + ansible.builtin.set_fact: + dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}" + changed_when: false + + - name: Initialize DEV2 passfile used + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + # ---------------------------- DIRECT LLDP IPv4 AUTH (10.x) ---------------------------- + - name: Try DEV2 login via direct LLDP IPv4 (10.x) with 'basicpass' + when: dev2_conn_method == "direct_lldp" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ lldp_dev2_ip }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_lldp_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select 'basicpass' for direct LLDP if previous login succeeded + when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + dev2_ssh_host: "{{ lldp_dev2_ip }}" + dev2_ssh_port: 22 + changed_when: false + + - name: Try DEV2 login via direct LLDP IPv4 (10.x) with 'basicpass2' (only if first failed) + when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ lldp_dev2_ip }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_lldp_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select 'basicpass2' for direct LLDP if second login succeeded + when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE" and dev2_lldp_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + dev2_ssh_host: "{{ lldp_dev2_ip }}" + dev2_ssh_port: 22 + changed_when: false + + - name: Fallback to tunnel if direct LLDP auth failed + when: dev2_conn_method == "direct_lldp" and dev2_passfile_used == "NONE" + delegate_to: localhost + ansible.builtin.set_fact: + dev2_conn_method: "tunnel" + changed_when: false + + - name: Debug connection method decision (new debug) + when: debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_conn_method={{ dev2_conn_method }}" + - "lldp_dev2_ip={{ lldp_dev2_ip | default('') }}" + - "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('') }}" + + # ---------------------------- TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ---------------------------- + - name: Add temporary IP on DEV1 (tolerate 'File exists') + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug result of adding temp IP to DEV1 + when: dev2_conn_method == "tunnel" + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + failed_when: false + + - name: Capture discovered DEV2 MAC (if any) + when: dev2_conn_method == "tunnel" + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC) + when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug ARP action summary on DEV1 + when: dev2_conn_method == "tunnel" + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Note skipping static ARP add (no MAC discovered) + when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0) + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + - name: Refresh ARP on DEV1’s LAN (best-effort) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + arping -U -I {{ arping_iface }} {{ dev2_side_ip.split('/')[0] }} -c 3 + changed_when: false + failed_when: false + + - name: Pick a free local TCP port for the tunnel (controller side) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port was found + when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0 + ansible.builtin.meta: end_host + + - name: Create control dir for SSH ControlMaster + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: "mktemp -d" + args: + executable: /bin/bash + register: mktemp_dir + changed_when: false + + - name: Record chosen local port and build ControlMaster socket path + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ mktemp_dir.stdout | trim }}" + _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl" + changed_when: false + + - name: Debug picked local port (controller) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.debug: + msg: + - "picked_local_port={{ _local_port }}" + - "ctrl_sock={{ _ctrl_sock }}" + - "dev1_host={{ ansible_host | default(inventory_hostname) }}" + + - name: Build tunnel remote target for DEV2 (prefer LLDP IPv6 when available) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.set_fact: + dev2_tunnel_target_host: >- + {{ + (lldp_dev2_ip6 is defined and (lldp_dev2_ip6 | string | trim | length > 0)) + | ternary( + '[' ~ (lldp_dev2_ip6 | string | trim) ~ '%' ~ (dev1_iface | string | trim) ~ ']', + dev2_host | string + ) + }} + dev2_tunnel_target_port: "{{ dev2_port | string }}" + changed_when: false + + - name: Debug tunnel remote target for DEV2 (IPv4/IPv6) + when: dev2_conn_method == "tunnel" and debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_tunnel_target={{ dev2_tunnel_target_host | default(dev2_host) }}:{{ dev2_tunnel_target_port | default(dev2_port) }}" + + - name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1 + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_tunnel_target_host | default(dev2_host) }}:{{ dev2_tunnel_target_port | default(dev2_port) }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Probe TCP reachability to DEV2 through the tunnel (nc) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Stop if tunnel TCP probe failed + when: dev2_conn_method == "tunnel" and nc_probe.rc != 0 + ansible.builtin.meta: end_host + + - name: Show passfiles available on controller (ls) + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles presence + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + - name: Try DEV2 login with 'basicpass' (root, tunnel) + when: dev2_conn_method == "tunnel" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select 'basicpass' if previous login succeeded (tunnel) + when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + dev2_ssh_host: "127.0.0.1" + dev2_ssh_port: "{{ _local_port }}" + changed_when: false + + - name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel) + when: dev2_conn_method == "tunnel" and dev2_passfile_used == "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + args: + executable: /bin/bash + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select 'basicpass2' if second login succeeded (tunnel) + when: dev2_conn_method == "tunnel" and dev2_passfile_used == "NONE" and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + dev2_ssh_host: "127.0.0.1" + dev2_ssh_port: "{{ _local_port }}" + changed_when: false + + # ---------------------------- FALLBACKS (Feb method style) ---------------------------- + - name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed) + when: + - dev2_conn_method == "tunnel" + - dev2_passfile_used == "NONE" + - (lldp_dev2_ip | default('')) is match('^10\\.') + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ lldp_dev2_ip }}" + if sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=20 \ + "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then + echo "basicpass"; exit 0 + fi + if sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=20 \ + "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then + echo "basicpass2"; exit 0 + fi + echo "NONE"; exit 1 + args: + executable: /bin/bash + register: dev2_auth_lldp4 + changed_when: false + failed_when: false + ignore_errors: true + + - name: Apply LLDP IPv4 fallback decision + when: + - dev2_conn_method == "tunnel" + - dev2_passfile_used == "NONE" + - (lldp_dev2_ip | default('')) is match('^10\\.') + - dev2_auth_lldp4 is defined + - (dev2_auth_lldp4.stdout | trim) in ['basicpass','basicpass2'] + delegate_to: localhost + ansible.builtin.set_fact: + dev2_conn_method: "lldp4_fallback" + dev2_passfile_used: "{{ dev2_auth_lldp4.stdout | trim }}" + dev2_ssh_host: "{{ lldp_dev2_ip }}" + dev2_ssh_port: 22 + changed_when: false + + - name: Debug final DEV2 connectivity decision (upgrade will use host/port/passfile below) + when: debugging | bool + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_conn_method={{ dev2_conn_method }}" + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_ssh_host={{ dev2_ssh_host | default('') }}" + - "dev2_ssh_port={{ dev2_ssh_port | default('') }}" + - "lldp_dev2_ip={{ lldp_dev2_ip | default('') }}" + - "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('') }}" + - "dev2_host={{ dev2_host }}:{{ dev2_port }}" + + # ============================ UPGRADE PROCEDURE BELOW (UNCHANGED) ============================ + + # ---------------------------- Read DEV2 hostname via chosen path ---------------------------- + - name: Read DEV2 hostname via selected connection (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare (inventory/DEV1/DEV2) + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug normalized hostnames (JSON-escaped) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # -------- New: DEV2 hostname classification (numeric ikejaNNNN vs other) -------- + - name: Detect DEV2 hostname pattern (ikeja or other) and raw form + ansible.builtin.set_fact: + _dev2_hn_is_ikeja_numeric: "{{ (_dev2_hn | default('')) is match('^ikeja[0-9]+$') }}" + _dev2_hn_raw: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') }}" + changed_when: false + + - name: Classify DEV2 hostname mismatch type + ansible.builtin.set_fact: + _hostname_softfail: "{{ _dev2_hn_is_ikeja_numeric and (_dev2_hn != _inv_hn) }}" + _hostname_warn: "{{ (not _dev2_hn_is_ikeja_numeric) and (_dev2_hn != _inv_hn) }}" + changed_when: false + + - name: Debug DEV2 hostname classification + delegate_to: localhost + ansible.builtin.debug: + msg: + - "_dev2_hn_is_ikeja_numeric={{ _dev2_hn_is_ikeja_numeric | default(false) }}" + - "_hostname_softfail={{ _hostname_softfail | default(false) }}" + - "_hostname_warn={{ _hostname_warn | default(false) }}" + - "_dev2_hn_raw={{ _dev2_hn_raw | default('') }}" + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Init soft-fail journal flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Append connection method decision to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Connection method for DEV2: ' ~ dev2_conn_method ~ (((dev2_conn_method == 'direct_lldp') | ternary(' (' ~ (dev2_ssh_host | default('')) ~ ')',''))) ] }}" + + + # ---- Apply hostname policy: soft-fail for other ikejaNNNN, warn+customfield for "weird" names ---- + - name: Soft-block if DEV2 hostname is another ikeja (probable wrong indoor) + when: _hostname_softfail | default(false) + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ (_journal | default([])) + [ 'DEV2 hostname looks like another ikeja-number device: dev2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ '. Treating as wrong indoor, skipping update.' ] }}" + + - name: Note non-standard DEV2 hostname and prepare indoor_hostname customfield + when: _hostname_warn | default(false) + ansible.builtin.set_fact: + indoor_hostname_value: "{{ '!' ~ (_dev2_hn_raw | default(_dev2_hn)) }}" + _journal: "{{ (_journal | default([])) + [ 'DEV2 hostname non-standard: ' ~ _dev2_hn_raw ~ ' (customfield indoor_hostname=' ~ ('!' ~ (_dev2_hn_raw | default(_dev2_hn))) ~ ')' ] }}" + + - name: Build indoor_hostname customfield payload + when: _hostname_warn | default(false) + delegate_to: localhost + ansible.builtin.set_fact: + indoor_hostname_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "customfield_set" + task_result: "indoor_hostname={{ indoor_hostname_value }}" + + - name: Publish indoor_hostname custom field to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'indoor_hostname', + 'task_result': indoor_hostname_value + } | to_json }}" + payload_encoding: "string" + changed_when: false + when: indoor_hostname_value is defined + + - name: Build specific image marker path on DEV2 (/tmp/prepared_for_) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Count existing preparation markers on DEV2 (best-effort) + when: dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if preparation markers already present on DEV2 + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + - name: Check bootenv-updated marker on DEV2 (/tmp/bootenv_updated) + when: dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "[ -f '/tmp/bootenv_updated' ] && echo PRESENT || echo ABSENT" + args: + executable: /bin/bash + register: dev2_bootenv_marker + changed_when: false + ignore_errors: true + + - name: Soft-block if bootenv-updated marker is present on DEV2 + when: dev2_bootenv_marker is defined + and (dev2_bootenv_marker.stdout | default('') | trim) == 'PRESENT' + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Bootenv updated marker present on DEV2 (/tmp/bootenv_updated). Skipping image staging/write' ] }}" + + - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 /usr/lib/release/firmux (if present) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug DEV2 firmux banner (if any) + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ====== INSERTED: two-step normalization preview (rev->r, then space->dash) ====== + - name: "Normalize DEV2 firmux (step 1): replace 'rev ' -> 'r' (debug only)" + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.set_fact: + _firmux_step1: "{{ (dev2_firmux.stdout | default('') | trim) | regex_replace('(?i)rev\\s+','r') }}" + changed_when: false + + - name: "Debug firmux after step 1 (rev->r)" + when: _firmux_step1 is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "_firmux_step1={{ _firmux_step1 }}" + + - name: "Normalize DEV2 firmux (step 2): replace space -> '-' (debug only)" + when: _firmux_step1 is defined + delegate_to: localhost + ansible.builtin.set_fact: + _firmux_step2: "{{ _firmux_step1 | replace(' ', '-') }}" + changed_when: false + + - name: "Debug firmux after step 2 (space->dash)" + when: _firmux_step2 is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "_firmux_step2={{ _firmux_step2 }}" + # ====== END INSERTED ====== + + # ===================== Early equality/substring check (same logic as checker) ===================== + - name: Normalize expected target step one compute base string (from image_filename) + delegate_to: localhost + ansible.builtin.set_fact: + expected_norm_step1: "{{ (image_filename | default('') | trim) }}" + changed_when: false + + - name: Extract version core X dot Y dot Z dash rNNNN from image_filename if present + delegate_to: localhost + ansible.builtin.set_fact: + expected_norm_core_list: "{{ (image_filename | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}" + changed_when: false + + - name: Choose first extracted core if available + delegate_to: localhost + ansible.builtin.set_fact: + expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}" + changed_when: false + + - name: Normalize expected target step two prefer extracted core when available + delegate_to: localhost + ansible.builtin.set_fact: + expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}" + changed_when: false + + - name: Capture firmware banner line from DEV2 (raw) + delegate_to: localhost + ansible.builtin.set_fact: + banner_raw: "{{ (dev2_firmux.stdout | default('') | trim) }}" + changed_when: false + + - name: Normalize banner line to X dot Y dot Z dash rNNNN suffix + delegate_to: localhost + ansible.builtin.set_fact: + banner_norm: >- + {{ + ((banner_raw | lower) is search('-r[0-9]+$')) + | ternary( + banner_raw, + (banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1')) + ) + }} + changed_when: false + + - name: Evaluate version match using normalized equality or substring + delegate_to: localhost + ansible.builtin.set_fact: + version_match: >- + {{ + (expected_norm | default('') | length > 0) + and ( + (banner_norm | default('')) == (expected_norm | default('')) + or ((banner_norm | default('') | lower) is search((expected_norm | default('') | lower))) + or ((expected_norm | default('') | lower) is search((banner_norm | default('') | lower))) + ) + }} + changed_when: false + + - name: Debug version compare snapshot (pre-write) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "expected_norm={{ expected_norm | default('') }}" + - "banner_raw={{ banner_raw | default('') }}" + - "banner_norm={{ banner_norm | default('') }}" + - "force_upgrade={{ force_upgrade | default(false) }}" + - "version_match={{ version_match | default(false) }}" + + - name: Soft-block if already on target (skip staging/write unless force-upgrade) + when: + - not (force_upgrade | default(false) | bool) + - version_match | default(false) + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ (_journal | default([])) + [ 'Device already running target image: ' ~ (banner_norm | default('')) ~ ' — skipping write/flip (use force-upgrade=yes to override)' ] }}" + expected_fw_core_early: "{{ expected_norm | default('') }}" + current_fw_core: "{{ banner_norm | default('') }}" + changed_when: false + + - name: Back-fill compare names for summary (no-op if already set) + when: not (_blocked | default(false)) + ansible.builtin.set_fact: + expected_fw_core_early: "{{ expected_fw_core_early | default(expected_norm | default('')) }}" + current_fw_core: "{{ current_fw_core | default(banner_norm | default('')) }}" + changed_when: false + + # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- + - name: Normalize rebootin (strict hours → base seconds/minutes) + delegate_to: localhost + ansible.builtin.set_fact: + reboot_hours: "{{ (rebootin | default('') | string | trim | int) }}" + reboot_seconds: "{{ (rebootin | default('') | string | trim | int) * 3600 }}" + reboot_minutes: "{{ (rebootin | default('') | string | trim | int) * 60 }}" + reboot_requested: true + _reboot_requested: true + + - name: Compute reboot delay (+20s grace) and mirror underscore vars + delegate_to: localhost + ansible.builtin.set_fact: + reboot_delay_seconds: "{{ (reboot_seconds | int) + 20 }}" + reboot_delay_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}" + _reboot_seconds: "{{ (reboot_seconds | int) + 20 }}" + _reboot_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}" + + - name: Debug reboot normalization detail + delegate_to: localhost + ansible.builtin.debug: + msg: + - "rebootin={{ rebootin | default('UNSET') }}" + - "reboot_hours={{ reboot_hours }}" + - "reboot_seconds={{ reboot_seconds }}" + - "reboot_minutes={{ reboot_minutes }}" + + - name: Warn if rebootin was not provided by consumer (debug only) + when: (rebootin | default('') | string | trim) == '' + delegate_to: localhost + ansible.builtin.debug: + msg: "WARNING: rebootin is empty or missing. Consumer likely did not pass -e rebootin=." + + - name: Debug reboot plan summary + delegate_to: localhost + ansible.builtin.debug: + msg: + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_seconds={{ _reboot_seconds | default(0) }}" + - "reboot_minutes={{ _reboot_minutes | default(0) }}" + + - name: Scheduler mode compute action_restart_timestamp + when: is_run_by_effective == 'scheduler' + delegate_to: localhost + ansible.builtin.set_fact: + action_restart_now_ts: "{{ lookup('pipe', 'date +%s') | int }}" + action_restart_timestamp: "{{ (lookup('pipe', 'date +%s') | int) + (_reboot_seconds | int) }}" + changed_when: false + + - name: Build human-readable reboot phrase + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_phrase: >- + {% if not (_reboot_requested | default(false)) -%} + reboot not requested + {%- elif (_reboot_seconds | int) <= 20 -%} + rebooting now + {%- elif (_reboot_seconds | int) >= 3600 -%} + reboot scheduled in {{ ((_reboot_seconds | int) // 3600) | int }}h + {%- else -%} + reboot scheduled in {{ (_reboot_minutes | int) }}m + {%- endif %} + + # ---------------------------- Journal: indoor start (we can proceed) ---------------------------- + - name: Build control queue payload for 'indoor start' journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting update. + conn_method={{ dev2_conn_method }}, + image={{ image_filename }}, + current firmware={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + dev2_hostname={{ _dev2_hn_raw | default(_dev2_hn) }}{% if _hostname_warn | default(false) %} (non-standard; indoor_hostname={{ indoor_hostname_value | default('') }}){% endif %}, + reboot planned in {{ ((_reboot_seconds | int) // 3600) | int }}h + when: dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish 'indoor start' journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 (no write yet) ---------------------------- + - name: Check local presence of image file on controller + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if local image is missing + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image (controller) + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch/unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + - name: Read fw_printenv size (line count) on DEV2 (soft health) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small (<30 lines) + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing DEV2 image md5 (NOFILE if missing) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -O \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@${HOST}:{{ (dev2_ssh_user == 'root') | ternary('/', '') }}{{ dev2_image_dir }}/{{ image_filename }}" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy (or if already present) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Soft-block if DEV2 md5 != expected + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Pre-validate image on DEV2 with 'update -c' (non-invasive) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=30 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug output from 'update -c' + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if 'update -c' did not return 'valid' + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for 'indoor aborted' journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish 'indoor aborted' journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP) + when: dev2_conn_method == "tunnel" + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and bank flip on DEV2 (guarded by soft-block) + when: not (_blocked | default(false)) + block: + + # --- supervised async write with stall handling --- + - name: Launch update -w on DEV2 (async) + delegate_to: localhost + vars: + _write_async_cap: 600 # 10 minutes max runtime (tune if needed) + _write_poll: 20 # poll every 20 seconds + block: + - name: Launch update -w on DEV2 (async) + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + async: "{{ _write_async_cap }}" + poll: "{{ _write_poll }}" + register: dev2_up_write + changed_when: true + failed_when: false + + - name: Evaluate async result + ansible.builtin.set_fact: + _updatew_stalled: >- + {{ (dev2_up_write.msg is defined) and + ('did not complete' in dev2_up_write.msg | lower) }} + _updatew_success: >- + {{ + ((dev2_up_write.stdout | default('')) is search('update is complete')) + or + ((not (dev2_up_write.skipped | default(false))) and ((((dev2_up_write.stdout | default('')) | trim) == '') and (((dev2_up_write.rc | default(1)) | int) == 0))) + }} + + - name: Journal stall if update -w timed out + when: _updatew_stalled | bool + ansible.builtin.set_fact: + journal_updatew_stalled: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "update -w has stalled. Rebooting device; please schedule another upgrade." + + - name: Publish stall journal to control queue + when: journal_updatew_stalled is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_updatew_stalled | to_json }}" + payload_encoding: "string" + register: rmq_stall_journal_resp + changed_when: (rmq_stall_journal_resp.json is defined) and (rmq_stall_journal_resp.json.routed | default(false) | bool) + failed_when: false + + - name: Schedule immediate reboot (20s delay) after stall + when: _updatew_stalled | bool + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "nohup /sbin/reboot -d 20 >/dev/null 2>&1 &" + args: + executable: /bin/bash + changed_when: true + ignore_errors: true + + - name: Stop play if update -w stalled + when: _updatew_stalled | bool + ansible.builtin.meta: end_play + + - name: Fail play if update -w completed but invalid output + when: (not _updatew_stalled) and (not _updatew_success) + ansible.builtin.fail: + msg: "update -w finished but did not report 'update is complete'" + + # --- continue normal path if not stalled --- + - name: Read current active partition on DEV2 (before flip) + when: not _updatew_stalled | bool + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 (flip 1↔2) + when: not _updatew_stalled | bool and dev2_active_before.stdout is defined + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Set fw_setenv active={{ dev2_new_active }} on DEV2 + when: not _updatew_stalled | bool and dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 (read back) + when: not _updatew_stalled | bool and dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 for this image + when: not _updatew_stalled | bool + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule DEV2 reboot after computed delay (seconds) + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ dev2_ssh_host }}" + PORT="{{ dev2_ssh_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Scheduler mode compute reboot_applied flag (gate for action_restart_timestamp) + when: is_run_by_effective == 'scheduler' + delegate_to: localhost + ansible.builtin.set_fact: + reboot_applied_cf: "{{ (dev2_reboot_sched is defined) and (dev2_reboot_sched.rc is defined) and ((dev2_reboot_sched.rc | int) == 0) }}" + changed_when: false + + - name: Scheduler mode set action_state waiting when reboot initiated + when: is_run_by_effective == 'scheduler' and (reboot_applied_cf | default(false)) + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_state', + 'task_result': 'waiting' + } | to_json }}" + + - name: Scheduler mode publish action_restart_timestamp custom field to control queue + when: is_run_by_effective == 'scheduler' and (reboot_applied_cf | default(false)) + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_restart_timestamp', + 'task_result': (action_restart_timestamp | string) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + + - name: Build 'indoor-restart-scheduled' tag payload + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish 'indoor-restart-scheduled' tag to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + - name: Note reboot was requested but value is invalid (format warning) + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Compute write success flag (_write_success) + ansible.builtin.set_fact: + _write_success: >- + {{ + ( + ( + (dev2_up_write.stdout | default('')) is search('update is complete') + ) + or + ( + ((not (dev2_up_write.skipped | default(false))) and (((dev2_up_write.stdout | default('')) | trim) == '') + and (((dev2_up_write.rc | default(1)) | int) == 0)) + ) + ) + if (dev2_up_write is defined) else false + }} + delegate_to: localhost + + - name: Debug write result and active banks (quick summary) + when: not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "write_success={{ _write_success | default(false) }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + + - name: Build 'indoor updated' journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 prepared for update, {{ _reboot_phrase }}. + conn_method={{ dev2_conn_method }}, + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: (_write_success | bool) + delegate_to: localhost + + - name: Publish 'indoor updated' journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + # ──────────────────────────────── Derive expected version for the checker (strict) ──────────────────────────────── + - name: Init after-upgrade scheduling vars (attempt=1, delay=reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ (reboot_seconds | int) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and UTC timestamp (for after-upgrade tracking) + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Derive expected target_version from image filename (strict X.Y.Z-rNNNN) + delegate_to: localhost + ansible.builtin.set_fact: + expected_fw_core: "{{ (image_filename | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default('', true) }}" + + - name: Debug derived expected target_version for checker + delegate_to: localhost + ansible.builtin.debug: + msg: + - "image_filename={{ image_filename }}" + - "expected_fw_core={{ expected_fw_core }}" + when: expected_fw_core is defined + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version_full: "{{ image_filename }}" + target_version: "{{ expected_fw_core }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + is_run_by: "{{ is_run_by_effective }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug after-upgrade plan (routing + delay + version) + ansible.builtin.debug: + msg: + - "routing_key={{ afterupgrade_routing_key }}" + - "x-delay(ms)={{ (au_delay_sec | int) * 1000 }}" + - "target_version={{ afterupgrade_payload.target_version | default('NA') }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Debug after-upgrade payload JSON (exactly what will be sent) + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ afterupgrade_payload | to_json }}" + when: afterupgrade_payload is defined + + - name: Publish delayed after-upgrade check (headers.x-delay) to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + # ---------------------------- Final operator summary (one-liners) ---------------------------- + - name: Summary key outcomes (one-liners) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_conn_method={{ dev2_conn_method }}" + - "dev2_ssh_host={{ dev2_ssh_host | default('') }}" + - "dev2_ssh_port={{ dev2_ssh_port | default('') }}" + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_hostname_raw={{ _dev2_hn_raw | default('') }}" + - "hostname_softfail={{ _hostname_softfail | default(false) }}" + - "hostname_warn={{ _hostname_warn | default(false) }}" + - "indoor_hostname_value={{ indoor_hostname_value | default('') }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "expected_fw_core_early={{ expected_fw_core_early | default('') }}" + - "current_fw_core={{ current_fw_core | default('') }}" + - "force_upgrade={{ force_upgrade | default(false) }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ reboot_requested | default(false) }}" + - "reboot_delay_seconds={{ reboot_delay_seconds if (reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_minutes={{ reboot_delay_minutes if (reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Scheduler mode report failure to NetBox when update needed but reboot not applied + when: is_run_by_effective == 'scheduler' and (((not (version_match | default(false))) or (force_upgrade | default(false)))) and (not ((dev2_reboot_sched is defined) and (dev2_reboot_sched.rc is defined) and ((dev2_reboot_sched.rc | int) == 0))) + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'action_state', + 'task_result': 'failed' + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Cleanup (always) + block: + - ansible.builtin.debug: + msg: "Entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close SSH ControlMaster (best-effort) + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove tunnel control dir (best-effort) + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address' and 'Address not available') + when: dev2_conn_method | default('') == "tunnel" + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + and ('Address not available' not in (del_ip.stdout | default(''))) + and ('Address not available' not in (del_ip.stderr | default(''))) + and ('RTNETLINK answers: Address not available' not in (del_ip.stdout | default(''))) + and ('RTNETLINK answers: Address not available' not in (del_ip.stderr | default(''))) + + + - name: Debug temp IP removal result + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined \ No newline at end of file diff --git a/files/fox200-2.2.2-r6820.bin b/files/fox200-2.2.2-r6820.bin new file mode 100644 index 0000000..3a3e372 Binary files /dev/null and b/files/fox200-2.2.2-r6820.bin differ