ansible-worker/files/ansible-playbooks/update-indoor-bootenv.yml

- name: Second-line indoor bootenv update via DEV1 → LLDP/tunnel → DEV2 (non-invasive control path)
  hosts: all
  gather_facts: no

  vars:
    # Busybox-safe PATH prefix for all remote raw calls on DEV1
    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "

    # DEV1 credentials (stable, like rebootin222)
    dev1_user: "root"
    dev1_pass: "wavewave"

    # Tunnel target DEV2 behind DEV1
    dev2_host: "192.168.1.1"
    dev2_port: 22

    # Temp IP we add to DEV1 so it can reach DEV2
    dev2_side_ip: "192.168.1.11/24"
    dev1_iface: "br-wan"

    # DEV2 behind the tunnel
    dev2_ssh_user: "root"
    dev2_passfiles:
      - "basicpass"
      - "basicpass2"

    # SSH options used from controller
    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"

    # Bootenv image to stage on DEV2
    bootenv_filename: "fox200_bootenv.bin"
    bootenv_sha256: "fcdea1909faff462a1d2791ddf513a52"

    # Aliases to reuse existing image_* logic (no firmware anymore, just bootenv)
    image_filename: "{{ bootenv_filename }}"
    image_md5: ""
    image_sha256: "{{ bootenv_sha256 }}"
    dev2_image_dir: "/tmp"
    dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"

    # Bootenv marker path on DEV2
    bootenv_marker: "/tmp/bootenv_updated"

    # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace).
    rebootin: ""

    # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"

    # ---------------- New: debugging toggle for newly added debug tasks ----------------
    debugging: true

    # ---------------- New: abstracted DEV2 connection (decided early) ----------------
    # "tunnel" (default) or "direct_lldp"
    dev2_conn_method: "tunnel"
    dev2_ssh_host: ""
    dev2_ssh_port: ""

  pre_tasks:
    # ------------------------------- Hostname sanity DEV1 -------------------------------
    - name: Read DEV1 hostname (busybox-safe)
      ansible.builtin.raw: >
        {{ pathprefix }}
        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
      register: dev1_host_read
      changed_when: false

    - name: Debug incoming parameters from consumer and defaults
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "rebootin={{ rebootin | default('UNSET') }}"
          - "bootenv_filename={{ bootenv_filename | default('UNSET') }}"
          - "bootenv_sha256={{ bootenv_sha256 | default('UNSET') }}"
          - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
          - "inventory_hostname={{ inventory_hostname }}"

    - name: Stop early if connected DEV1 hostname != inventory
      ansible.builtin.meta: end_host
      when: (dev1_host_read.stdout | trim | length > 0) and
            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))

  tasks:
    # ============================ LLDP-FIRST CONNECTION DECISION ============================
    - name: Compute hostname digits key for LLDP lookup (DEV2)
      ansible.builtin.set_fact:
        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
      changed_when: false

    - name: Discover DEV2 candidate IP via LLDP on DEV1
      ansible.builtin.raw: >
        {{ pathprefix }}
        DIGITS="{{ dev2_lldp_digits }}";
        cat /var/run/lldp_server.json 2>/dev/null \
          | grep "${DIGITS}" -A 10 \
          | grep address \
          | grep -vE 'subtype|ipv6' \
          | awk -F'"' '{ print $4 }' \
          | head -n1
      register: dev2_lldp_ip_raw
      changed_when: false
      failed_when: false

    - name: Capture LLDP-derived DEV2 IP (if any)
      ansible.builtin.set_fact:
        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
      changed_when: false

    - name: Classify LLDP candidate range
      delegate_to: localhost
      ansible.builtin.set_fact:
        lldp_ip_class: >-
          {% set ip = (lldp_dev2_ip | default('')) %}
          {% if ip == '' %}none
          {% elif ip.startswith('10.') %}10
          {% elif ip.startswith('192.168.') %}192_168
          {% else %}other{% endif %}
      changed_when: false

    - name: Debug LLDP candidate and classification (new debug)
      when: debugging | bool
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
          - "LLDP candidate IP={{ lldp_dev2_ip | default('<none>') }}"
          - "LLDP class={{ lldp_ip_class | default('none') }}"

    # Decide dev2_conn_method = direct_lldp for 10.x, tunnel otherwise
    - name: Set connection method to tunnel by default
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_conn_method: "tunnel"
      changed_when: false

    - name: Switch to direct LLDP mode for 10.x.x.x
      when: lldp_ip_class == "10"
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_conn_method: "direct_lldp"
      changed_when: false

    - name: Debug connection method decision (new debug)
      when: debugging | bool
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "dev2_conn_method={{ dev2_conn_method }}"
          - "lldp_dev2_ip={{ lldp_dev2_ip | default('<none>') }}"

    # Direct LLDP 10.x login (no tunnel, reusing two-step auth style)
    - name: Try DEV2 login via direct LLDP IP with 'basicpass' (10.x)
      when: dev2_conn_method == "direct_lldp"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ lldp_dev2_ip }}"
        sshpass -f basicpass ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
      args:
        executable: /bin/bash
      register: dev2_lldp_try_basicpass
      changed_when: false
      ignore_errors: true

    - name: Select 'basicpass' for direct LLDP if previous login succeeded
      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_passfile_used: "basicpass"
      changed_when: false

    - name: Try DEV2 login via direct LLDP IP with 'basicpass2' (10.x, only if first failed)
      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used is not defined)
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ lldp_dev2_ip }}"
        sshpass -f basicpass2 ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
      args:
        executable: /bin/bash
      register: dev2_lldp_try_basicpass2
      changed_when: false
      ignore_errors: true

    - name: Select 'basicpass2' for direct LLDP if second login succeeded
      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined and dev2_lldp_try_basicpass2.rc == 0
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_passfile_used: "basicpass2"
      changed_when: false

    - name: Mark DEV2 auth as NONE for direct LLDP if both attempts failed
      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_passfile_used: "NONE"
      changed_when: false

    - name: Debug direct LLDP auth decision (new debug)
      when: dev2_conn_method == "direct_lldp" and debugging | bool
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "dev2_lldp_try_basicpass.rc={{ dev2_lldp_try_basicpass.rc | default('NA') }}"
          - "dev2_lldp_try_basicpass2.rc={{ dev2_lldp_try_basicpass2.rc | default('NA') }}"
          - "dev2_passfile_used={{ dev2_passfile_used | default('NONE') }}"
          - "lldp_dev2_ip={{ lldp_dev2_ip | default('<none>') }}"

    # Set host/port for direct LLDP if auth succeeded
    - name: Set direct LLDP DEV2 SSH host/port
      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used != "NONE"
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_ssh_host: "{{ lldp_dev2_ip }}"
        dev2_ssh_port: 22
      changed_when: false

    # ---------------------------- Idempotent temp IP on DEV1 (TUNNEL ONLY) ----------------------------
    - name: Add temporary IP on DEV1 (tolerate 'File exists')
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: >
        {{ pathprefix }}
        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
      register: add_ip
      changed_when: add_ip.rc == 0
      failed_when: >
        add_ip.rc != 0
        and ('File exists' not in (add_ip.stdout | default('')))
        and ('File exists' not in (add_ip.stderr | default('')))

    - name: Debug result of adding temp IP to DEV1
      when: dev2_conn_method == "tunnel"
      ansible.builtin.debug:
        msg:
          - "add_ip.rc={{ add_ip.rc | default('') }}"
          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"

    # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 (TUNNEL) ----------------------------
    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: >
        {{ pathprefix }}
        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
      register: dev2_mac_scan
      changed_when: false

    - name: Capture discovered DEV2 MAC (if any)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.set_fact:
        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
      changed_when: false

    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: >
        {{ pathprefix }}
        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
      register: dev2_arp_del
      changed_when: false
      failed_when: false

    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: >
        {{ pathprefix }}
        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
      register: dev2_arp_add
      changed_when: dev2_arp_add.rc == 0
      failed_when: >
        dev2_arp_add.rc != 0
        and ('File exists' not in (dev2_arp_add.stdout | default('')))
        and ('File exists' not in (dev2_arp_add.stderr | default('')))

    - name: Debug  ARP action summary on DEV1
      when: dev2_conn_method == "tunnel"
      ansible.builtin.debug:
        msg:
          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"

    - name: Note  skipping static ARP add (no MAC discovered)
      when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0)
      ansible.builtin.debug:
        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"

    # ---------------------------- Local tunnel preparation (TUNNEL ONLY) ----------------------------
    - name: Pick a free local TCP port for the tunnel (controller side)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        pick() {
            for i in $(seq 1 25); do
                p="$(shuf -i 20000-39999 -n 1)"
                if command -v ss >/dev/null 2>&1; then
                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
                        echo "$p"; return 0
                    fi
                else
                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
                        echo "$p"; return 0
                    fi
                fi
            done
            return 1
        }
        pick
      register: pick_port
      changed_when: false

    # (moved up) Stop immediately if no free local port was found
    - name: Stop if no free local port was found
      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
      ansible.builtin.meta: end_host

    # (moved up) Set chosen port and control socket path
    - name: Record chosen local port and create control dir for SSH ControlMaster
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.set_fact:
        _local_port: "{{ pick_port.stdout | trim }}"
        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"

    - name: Build path for SSH ControlMaster socket
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.set_fact:
        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"

    - name: Debug picked local port (controller)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "picked_local_port={{ _local_port }}"
          - "ctrl_sock={{ _ctrl_sock }}"
          - "dev1_host={{ ansible_host | default(inventory_hostname) }}"
          - "dev2_target={{ dev2_host }}:{{ dev2_port }}"

    - name: Show current listeners on picked port (ss/lsof)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        P="{{ _local_port }}"
        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p'
        { lsof -nP -iTCP:"{{ _local_port }}" -sTCP:LISTEN 2>/dev/null || true; }
      register: port_listeners_before
      changed_when: false
      failed_when: false

    - name: Debug listeners on picked port (before starting tunnel)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "listeners_before:\n{{ (port_listeners_before.stdout | default('')) | trim }}"

    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3

    # ---------------------------- Start SSH local forward via DEV1 (TUNNEL ONLY) ----------------------------
    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        USER="{{ dev1_user }}"
        HOST="{{ ansible_host | default(inventory_hostname) }}"
        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
            -M -S "{{ _ctrl_sock }}" \
            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
            "${USER}@${HOST}"
      args:
        executable: /bin/bash
      register: start_tunnel
      changed_when: true

    - name: Debug ControlMaster start result (rc/stdout/stderr)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "start_tunnel.rc={{ start_tunnel.rc | default('NA') }}"
          - "start_tunnel.stdout={{ (start_tunnel.stdout | default('')) | trim }}"
          - "start_tunnel.stderr={{ (start_tunnel.stderr | default('')) | trim }}"

    - name: Show who is listening now on the local port (post-start)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        P="{{ _local_port }}"
        echo "== ss -ltnp on :${P} =="
        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
        echo "== lsof LISTEN on :${P} =="
        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
      register: port_listeners_after
      changed_when: false
      failed_when: false

    - name: Debug listeners on picked port (after starting tunnel)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "{{ (port_listeners_after.stdout | default('')) | trim }}"

    - name: Wait a moment for tunnel to settle
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.wait_for:
        timeout: 1
      changed_when: false

    - name: Verify ControlMaster is running (ssh -O check)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ ansible_host | default(inventory_hostname) }}"
        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
      register: tun_check
      changed_when: false

    - name: Debug  ControlMaster status
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "tunnel_check.rc={{ tun_check.rc }}"
          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"

    - name: Debug  ControlMaster check (full rc/stdout/stderr)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "tun_check.rc={{ tun_check.rc | default('NA') }}"
          - "tun_check.stdout={{ (tun_check.stdout | default('')) | trim }}"
          - "tun_check.stderr={{ (tun_check.stderr | default('')) | trim }}"

    # ---------------------------- Controller-side sanity for DEV2 auth (TUNNEL) ----------------------------
    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        nc -z -w5 127.0.0.1 "{{ _local_port }}"
      register: nc_probe
      changed_when: false
      ignore_errors: true

    - name: Debug  tunnel reachability result
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "nc.rc={{ nc_probe.rc }}"
          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"

    - name: Stop if tunnel TCP probe failed
      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
      ansible.builtin.meta: end_host

    - name: Show passfiles available on controller (ls)
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
      register: dev2_ls
      changed_when: false

    - name: Debug  passfiles presence
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "{{ (dev2_ls.stdout | default('')) | trim }}"
          - "{{ (dev2_ls.stderr | default('')) | trim }}"

    - name: Refresh ARP 2 on DEV1’s LAN (send unsolicited ARP from temporary IP)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3

    # ---------------------------- Single banner probe (kept, TUNNEL ONLY) ----------------------------
    - name: Probe SSH banner through tunnel (pre-auth, quick)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        PORT="{{ _local_port }}"
        ssh -p "$PORT" \
          -o PreferredAuthentications=none \
          -o PubkeyAuthentication=no \
          -o KbdInteractiveAuthentication=no \
          -o PasswordAuthentication=no \
          -o NumberOfPasswordPrompts=0 \
          -o ConnectTimeout=5 \
          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
          -vvv root@127.0.0.1 true 2>&1 || true
      register: tunnel_banner_probe
      changed_when: false
      failed_when: false

    - name: Debug  SSH preauth probe (first 40 lines)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg: "{{ (tunnel_banner_probe.stdout | default('') | split('\n'))[:40] | join('\n') }}"

    # ---------------------------- Pick DEV2 password for root (TUNNEL) ----------------------------
    - name: Try DEV2 login with 'basicpass' (root, tunnel)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        PORT="{{ _local_port }}"
        sshpass -f basicpass ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
      args:
        executable: /bin/bash
      register: dev2_try_basicpass
      changed_when: false
      ignore_errors: true

    - name: Snapshot listeners on local tunnel port (after basicpass try)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        P="{{ _local_port }}"
        echo "== ss -ltnp on :${P} =="
        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
        echo "== lsof LISTEN on :${P} =="
        { lsof -nP -iTCP:"{{ _local_port }}" -sTCP:LISTEN 2>/dev/null || true; }
        echo "== ps/grep ControlMaster by ControlPath =="
        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
      register: listeners_after_basicpass
      changed_when: false
      failed_when: false

    - name: Debug  auth try context (basicpass)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "auth_try=basicpass rc={{ dev2_try_basicpass.rc | default('NA') }}"
          - "local_port={{ _local_port }}"
          - "ctrl_sock={{ _ctrl_sock }}"
          - "listeners:\n{{ (listeners_after_basicpass.stdout | default('')) | trim }}"

    - name: Select 'basicpass' if previous login succeeded (tunnel)
      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_passfile_used: "basicpass"
      changed_when: false

    - name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel)
      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        PORT="{{ _local_port }}"
        sshpass -f basicpass2 ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
      args:
        executable: /bin/bash
      register: dev2_try_basicpass2
      changed_when: false
      ignore_errors: true

    - name: Snapshot listeners on local tunnel port (after basicpass2 try)
      when: dev2_conn_method == "tunnel"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        P="{{ _local_port }}"
        echo "== ss -ltnp on :${P} =="
        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
        echo "== lsof LISTEN on :${P} =="
        { lsof -nP -iTCP:"{{ _local_port }}" -sTCP:LISTEN 2>/dev/null || true; }
        echo "== ps/grep ControlMaster by ControlPath =="
        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
      register: listeners_after_basicpass2
      changed_when: false
      failed_when: false

    - name: Debug  auth try context (basicpass2)
      when: dev2_conn_method == "tunnel" and dev2_try_basicpass2 is defined
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "auth_try=basicpass2 rc={{ dev2_try_basicpass2.rc | default('NA') }}"
          - "local_port={{ _local_port }}"
          - "ctrl_sock={{ _ctrl_sock }}"
          - "listeners:\n{{ (listeners_after_basicpass2.stdout | default('')) | trim }}"

    - name: Select 'basicpass2' if second login succeeded (tunnel)
      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_passfile_used: "basicpass2"
      changed_when: false

    - name: Mark DEV2 auth as NONE if both attempts failed (tunnel)
      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_passfile_used: "NONE"
      changed_when: false

    - name: Debug  selected DEV2 passfile
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "dev2_passfile_used={{ dev2_passfile_used }}"
          - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}"
          - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}"

    # ---------------------------- Set abstracted host/port for tunnel mode ----------------------------
    - name: Set DEV2 SSH host/port for tunnel mode
      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE"
      delegate_to: localhost
      ansible.builtin.set_fact:
        dev2_ssh_host: "127.0.0.1"
        dev2_ssh_port: "{{ _local_port }}"
      changed_when: false

    # ---------------------------- Read DEV2 hostname via chosen path ----------------------------
    - name: Read DEV2 hostname via selected connection (busybox-safe)
      when: dev2_passfile_used != "NONE"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
      args:
        executable: /bin/bash
      register: dev2_host_read
      changed_when: false

    - name: Normalize hostnames for strict compare (inventory/DEV1/DEV2)
      ansible.builtin.set_fact:
        _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
        _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
        _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"

    - name: Debug  normalized hostnames (JSON-escaped)
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "inv={{ _inv_hn | tojson }}"
          - "dev1={{ _dev1_hn | tojson }}"
          - "dev2={{ _dev2_hn | tojson }}"

    # ---------------- Hostname equality guard (soft-journal and stop) ----------------
    - name: Guard DEV2 hostname must equal inventory AND DEV1 (prevents IP churn mistakes)
      block:
        - name: Fail if DEV2 hostname differs from inventory/DEV1
          ansible.builtin.fail:
            msg: >
              Hostname mismatch: DEV2='{{ _dev2_hn }}',
              inventory='{{ _inv_hn }}',
              DEV1='{{ _dev1_hn }}'
          when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
      rescue:
        - name: Initialize journal array for hostname mismatch
          ansible.builtin.set_fact:
            _journal: []
            _blocked: true
            _prep_blocked: false
          delegate_to: localhost

        - name: Append hostname mismatch info to journal
          ansible.builtin.set_fact:
            _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}"
          delegate_to: localhost

        - name: Build control queue payload for indoor aborted journal (hostname check)
          ansible.builtin.set_fact:
            journal_indoor_aborted:
              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
              task_name: "journal_add"
              task_result: >-
                indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
          delegate_to: localhost

        - name: Publish indoor aborted journal (hostname mismatch)
          ansible.builtin.uri:
            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
            method: POST
            user: "{{ rmq_user }}"
            password: "{{ rmq_pass }}"
            force_basic_auth: true
            status_code: 200
            headers:
              content-type: "application/json"
            body_format: json
            body:
              properties:
                content_type: "application/json"
              routing_key: "{{ control_queue }}"
              payload: "{{ journal_indoor_aborted | to_json }}"
              payload_encoding: "string"
          register: rmq_journal_indoor_aborted_hn_resp
          changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
          failed_when: >
            (rmq_journal_indoor_aborted_hn_resp.status != 200) or
            (rmq_journal_indoor_aborted_hn_resp.json is not defined) or
            (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
          delegate_to: localhost

        - name: Stop host after hostname mismatch
          ansible.builtin.meta: end_host

    # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
    - name: Init soft-fail journal flags
      ansible.builtin.set_fact:
        _journal: []
        _prep_blocked: false
        _blocked: false

    - name: Append connection method decision to journal
      ansible.builtin.set_fact:
        _journal: "{{ _journal + [ 'Connection method for DEV2: ' ~ dev2_conn_method ~ (dev2_conn_method == 'direct_lldp' | ternary(' (' ~ (dev2_ssh_host | default('')) ~ ')','')) ] }}"

    - name: Build bootenv marker path on DEV2
      ansible.builtin.set_fact:
        _marker_specific: "{{ bootenv_marker }}"

    - name: Count existing preparation markers on DEV2 (best-effort)
      when: dev2_passfile_used != "NONE"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=10 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "ls /tmp/prepared_for* 2>/dev/null | wc -l"
      args:
        executable: /bin/bash
      register: dev2_prep_count
      changed_when: false
      ignore_errors: true

    - name: Soft-block if preparation markers already present on DEV2
      when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
      ansible.builtin.set_fact:
        _prep_blocked: true
        _blocked: true
        _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping bootenv update' ] }}"

    - name: Check if bootenv marker already present on DEV2
      when: dev2_passfile_used != "NONE"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=10 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "[ -f '{{ _marker_specific }}' ] && echo PRESENT || echo ABSENT"
      args:
        executable: /bin/bash
      register: dev2_bootenv_marker_state
      changed_when: false
      ignore_errors: true

    - name: Soft-block if bootenv marker already present
      when: dev2_bootenv_marker_state is defined and (dev2_bootenv_marker_state.stdout | default('') | trim) == 'PRESENT'
      ansible.builtin.set_fact:
        _prep_blocked: true
        _blocked: true
        _journal: "{{ _journal + [ 'Bootenv marker already present on DEV2; skipping bootenv update' ] }}"

    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3

    # ---------------------------- DEV2 version firmux primary check ----------------------------
    - name: Read DEV2 /usr/lib/release/firmux (if present)
      when: dev2_passfile_used != "NONE"
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "cat /usr/lib/release/firmux 2>/dev/null || true"
      args:
        executable: /bin/bash
      register: dev2_firmux
      changed_when: false
      ignore_errors: true

    - name: Debug  DEV2 firmux banner (if any)
      when: dev2_firmux is defined
      delegate_to: localhost
      ansible.builtin.debug:
        msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"

    # ---------------------------- Normalize rebootin early (HOURS) ----------------------------
    - name: Normalize rebootin (strict hours → base seconds/minutes)
      delegate_to: localhost
      ansible.builtin.set_fact:
        reboot_hours: "{{ (rebootin | default('') | string | trim | int) }}"
        reboot_seconds: "{{ (rebootin | default('') | string | trim | int) * 3600 }}"
        reboot_minutes: "{{ (rebootin | default('') | string | trim | int) * 60 }}"
        reboot_requested: true
        _reboot_requested: true

    - name: Compute reboot delay (+20s grace) and mirror underscore vars
      delegate_to: localhost
      ansible.builtin.set_fact:
        reboot_delay_seconds: "{{ (reboot_seconds | int) + 20 }}"
        reboot_delay_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}"
        _reboot_seconds: "{{ (reboot_seconds | int) + 20 }}"
        _reboot_minutes: "{{ (((reboot_seconds | int) + 20) // 60) | int }}"

    - name: Debug reboot normalization detail
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "rebootin={{ rebootin | default('UNSET') }}"
          - "reboot_hours={{ reboot_hours }}"
          - "reboot_seconds={{ reboot_seconds }}"
          - "reboot_minutes={{ reboot_minutes }}"

    - name: Warn if rebootin was not provided by consumer (debug only)
      when: (rebootin | default('') | string | trim) == ''
      delegate_to: localhost
      ansible.builtin.debug:
        msg: "WARNING: rebootin is empty or missing. Consumer likely did not pass -e rebootin=<hours>."

    - name: Debug  reboot plan summary
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "reboot_requested={{ _reboot_requested | default(false) }}"
          - "reboot_seconds={{ _reboot_seconds | default(0) }}"
          - "reboot_minutes={{ _reboot_minutes | default(0) }}"

    - name: Build human-readable reboot phrase
      delegate_to: localhost
      ansible.builtin.set_fact:
        _reboot_phrase: >-
          {% if not (_reboot_requested | default(false)) -%}
          reboot not requested
          {%- elif (_reboot_seconds | int) <= 20 -%}
          rebooting now
          {%- elif (_reboot_seconds | int) >= 3600 -%}
          reboot scheduled in {{ ((_reboot_seconds | int) // 3600) | int }}h
          {%- else -%}
          reboot scheduled in {{ (_reboot_minutes | int) }}m
          {%- endif %}

    # ---------------------------- Journal: indoor start (we can proceed) ----------------------------
    - name: Build control queue payload for 'indoor start' journal
      ansible.builtin.set_fact:
        journal_indoor_start:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            Indoor: Dev2 is reachable, starting bootenv update.
            conn_method={{ dev2_conn_method }},
            bootenv={{ bootenv_filename }},
            current firmware={{ (dev2_firmux.stdout | default('unknown')) | trim }},
            reboot planned in {{ ((_reboot_seconds | int) // 3600) | int }}h
      when: dev2_passfile_used != "NONE"
      delegate_to: localhost

    - name: Publish 'indoor start' journal to control queue
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_indoor_start | to_json }}"
          payload_encoding: "string"
      register: rmq_journal_indoor_start_resp
      changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
      failed_when: >
        (rmq_journal_indoor_start_resp.status != 200) or
        (rmq_journal_indoor_start_resp.json is not defined) or
        (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
      when: journal_indoor_start is defined
      delegate_to: localhost

    # ---------------------------- Stage bootenv on DEV2 (no write yet) ----------------------------
    - name: Check local presence of bootenv file on controller
      delegate_to: localhost
      ansible.builtin.stat:
        path: "{{ bootenv_filename }}"
      register: local_img

    - name: Soft-block if local bootenv image is missing
      when: not local_img.stat.exists
      ansible.builtin.set_fact:
        _blocked: true
        _journal: "{{ _journal + [ 'Local bootenv image missing on controller: ' ~ bootenv_filename ] }}"

    - name: Compute local sha256 of the bootenv image (controller)
      when: local_img.stat.exists
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        md5sum "{{ bootenv_filename }}" | awk '{print $1}'
      register: local_sha256
      changed_when: false
      ignore_errors: true

    - name: Soft-block if local bootenv sha256 mismatch/unavailable
      when: local_img.stat.exists and (bootenv_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (bootenv_sha256 | trim))
      ansible.builtin.set_fact:
        _prep_blocked: true
        _blocked: true
        _journal: "{{ _journal + [ 'Local bootenv sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (bootenv_sha256 | trim) ] }}"

    - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3

    # fw_printenv health before bootenv update (inverted logic vs firmware play)
    - name: Read fw_printenv size (line count) on DEV2 (soft health)
      when: dev2_passfile_used != "NONE" and not (_blocked | default(false))
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=10 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "(fw_printenv 2>/dev/null | wc -l) || echo 0"
      args:
        executable: /bin/bash
      register: dev2_fwenv_wc
      changed_when: false
      ignore_errors: true

    - name: Soft-block if fw_printenv already looks healthy (>=30 lines)
      when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) >= 30)
      ansible.builtin.set_fact:
        _prep_blocked: true
        _blocked: true
        _journal: "{{ _journal + [ 'fw_printenv already has ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (>=30). Skipping bootenv update' ] }}"

    # ---------------------------- Bootenv staging on DEV2 ----------------------------
    - name: Check existing bootenv sha256 on DEV2 (NOSHA if missing)
      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
      args:
        executable: /bin/bash
      register: dev2_sha256_before
      changed_when: false

    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3

    - name: Copy bootenv image to DEV2 if missing or sha256 mismatch
      when: local_img.stat.exists
            and dev2_passfile_used != "NONE"
            and not (_blocked | default(false))
            and ((dev2_sha256_before.stdout | trim) != (bootenv_sha256 | trim))
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o AddressFamily=inet \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "curl -L --fail --silent --show-error -k -o '{{ dev2_image_path }}' 'https://cloud.ikeja.co.za/firmwares/uboot/{{ bootenv_filename }}'"
      args:
        executable: /bin/bash
      register: scp_bootenv
      changed_when: true

    - name: Compute sha256 of bootenv image on DEV2 after copy (or if already present)
      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=30 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
          "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
      args:
        executable: /bin/bash
      register: dev2_sha256_after
      changed_when: false
      ignore_errors: true

    - name: Soft-block if DEV2 bootenv sha256 mismatch/unavailable
      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (bootenv_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (bootenv_sha256 | trim))
      ansible.builtin.set_fact:
        _prep_blocked: true
        _blocked: true
        _journal: "{{ _journal + [ 'Remote bootenv sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (bootenv_sha256 | trim) ] }}"

    # ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
    - name: Build control queue payload for 'indoor aborted' journal
      ansible.builtin.set_fact:
        journal_indoor_aborted:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
      when: (_blocked | default(false))
      delegate_to: localhost

    - name: Publish 'indoor aborted' journal to control queue
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_indoor_aborted | to_json }}"
          payload_encoding: "string"
      register: rmq_journal_indoor_aborted_resp
      changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
      failed_when: >
        (rmq_journal_indoor_aborted_resp.status != 200) or
        (rmq_journal_indoor_aborted_resp.json is not defined) or
        (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
      when: journal_indoor_aborted is defined
      delegate_to: localhost

    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
      when: dev2_conn_method == "tunnel"
      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3

    # ============================ ACTUAL BOOTENV WRITE (only if not blocked) ============================
    - name: Upgrade bootenv on DEV2 (guarded by soft-block)
      when: not (_blocked | default(false))
      block:

        - name: Write bootenv to /dev/mtdblock1 on DEV2 (dd)
          delegate_to: localhost
          ansible.builtin.shell: |
            set -e
            HOST="{{ dev2_ssh_host }}"
            PORT="{{ dev2_ssh_port }}"
            sshpass -f "{{ dev2_passfile_used }}" ssh \
              -o AddressFamily=inet \
              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
              -o ConnectTimeout=30 \
              -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
              "dd if='{{ dev2_image_path }}' of=/dev/mtdblock1"
          args:
            executable: /bin/bash
          register: dev2_bootenv_dd
          changed_when: true
          failed_when: dev2_bootenv_dd.rc != 0

        - name: Debug dd output (bootenv)
          delegate_to: localhost
          ansible.builtin.debug:
            msg:
              - "dd.rc={{ dev2_bootenv_dd.rc | default('NA') }}"
              - "dd.stdout={{ (dev2_bootenv_dd.stdout | default('')) | trim }}"
              - "dd.stderr={{ (dev2_bootenv_dd.stderr | default('')) | trim }}"

        - name: Run sync on DEV2 after bootenv write
          delegate_to: localhost
          ansible.builtin.shell: |
            set -e
            HOST="{{ dev2_ssh_host }}"
            PORT="{{ dev2_ssh_port }}"
            sshpass -f "{{ dev2_passfile_used }}" ssh \
              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
              -o ConnectTimeout=30 \
              -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
              "sync"
          args:
            executable: /bin/bash
          register: dev2_bootenv_sync
          changed_when: true
          failed_when: dev2_bootenv_sync.rc != 0

        - name: Debug sync output (bootenv)
          delegate_to: localhost
          ansible.builtin.debug:
            msg:
              - "sync.rc={{ dev2_bootenv_sync.rc | default('NA') }}"
              - "sync.stdout={{ (dev2_bootenv_sync.stdout | default('')) | trim }}"
              - "sync.stderr={{ (dev2_bootenv_sync.stderr | default('')) | trim }}"

        - name: Set bootenv write success flag
          delegate_to: localhost
          ansible.builtin.set_fact:
            _bootenv_write_success: "{{ (dev2_bootenv_dd.rc | default(1)) == 0 and (dev2_bootenv_sync.rc | default(1)) == 0 }}"

        - name: Create bootenv marker on DEV2
          delegate_to: localhost
          ansible.builtin.shell: |
            set -e
            HOST="{{ dev2_ssh_host }}"
            PORT="{{ dev2_ssh_port }}"
            sshpass -f "{{ dev2_passfile_used }}" ssh \
              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
              -o ConnectTimeout=10 \
              -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
              "touch '{{ _marker_specific }}'"
          args:
            executable: /bin/bash
          register: dev2_marker_write
          changed_when: true
          ignore_errors: true

    # ---------------------------- Reboot scheduling (normalized) ----------------------------
    - name: Schedule DEV2 reboot after computed delay (seconds)
      when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        HOST="{{ dev2_ssh_host }}"
        PORT="{{ dev2_ssh_port }}"
        SECS="{{ _reboot_seconds | int }}"
        CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
          -o ConnectTimeout=10 \
          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" "${CMD}"
      args:
        executable: /bin/bash
      register: dev2_reboot_sched
      changed_when: true
      ignore_errors: true

    - name: Build 'indoor-restart-scheduled' tag payload
      ansible.builtin.set_fact:
        tag_restart_sched_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "tag_add"
          task_result: "indoor-restart-scheduled"
      when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
      delegate_to: localhost

    - name: Publish 'indoor-restart-scheduled' tag to control queue
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ tag_restart_sched_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_tag_restart_sched_resp
      changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)
      failed_when: >
        (rmq_tag_restart_sched_resp.status != 200) or
        (rmq_tag_restart_sched_resp.json is not defined) or
        (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool))
      when: tag_restart_sched_payload is defined
      delegate_to: localhost

    - name: Note  reboot was requested but value is invalid (format warning)
      when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
      ansible.builtin.debug:
        msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"

    # ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
    - name: Compute write success flag (_write_success)
      ansible.builtin.set_fact:
        _write_success: "{{ _bootenv_write_success | default(false) }}"
      when: not (_blocked | default(false))
      delegate_to: localhost

    - name: Debug  write result (bootenv)
      when: not (_blocked | default(false))
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "bootenv_write_success={{ _write_success | default(false) }}"

    - name: Build 'indoor updated' journal payload text
      ansible.builtin.set_fact:
        journal_indoor_updated:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            Indoor: Dev2 bootenv updated, {{ _reboot_phrase }}.
            conn_method={{ dev2_conn_method }},
            write_done={{ _write_success }},
            active_before=NA,
            active_after=NA
      when: not (_blocked | default(false)) and (_write_success | bool)
      delegate_to: localhost

    - name: Publish 'indoor updated' journal to control queue
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_indoor_updated | to_json }}"
          payload_encoding: "string"
      register: rmq_journal_indoor_updated_resp
      changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
      failed_when: >
        (rmq_journal_indoor_updated_resp.status != 200) or
        (rmq_journal_indoor_updated_resp.json is not defined) or
        (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
      when: journal_indoor_updated is defined
      delegate_to: localhost

    # ──────────────────────────────── After-upgrade scheduling (same mechanism, bootenv-focused) ────────────────────────────────
    - name: Init after-upgrade scheduling vars (attempt=1, delay=reboot + 6m)
      ansible.builtin.set_fact:
        au_attempt: 1
        au_max_attempts: 3
        au_delay_sec: "{{ (reboot_seconds | int) + 360 }}"
      when: not (_blocked | default(false)) and (_write_success | bool)
      delegate_to: localhost

    - name: Generate correlation ID and UTC timestamp (for after-upgrade tracking)
      ansible.builtin.set_fact:
        au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
        au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
      when: au_delay_sec is defined
      delegate_to: localhost

    - name: Derive expected target_version from bootenv context (placeholder)
      delegate_to: localhost
      ansible.builtin.set_fact:
        expected_fw_core: ""
      when: au_delay_sec is defined

    - name: Debug  derived expected target_version for checker
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "bootenv_filename={{ bootenv_filename }}"
          - "expected_fw_core={{ expected_fw_core }}"
      when: expected_fw_core is defined

    - name: Build after-upgrade check payload (attempt 1)
      ansible.builtin.set_fact:
        afterupgrade_payload:
          task_name: "afterupgrade_indoor_check"
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          target_version_full: "{{ bootenv_filename }}"
          target_version: "{{ expected_fw_core }}"
          attempt: "{{ au_attempt }}"
          max_attempts: "{{ au_max_attempts }}"
          current_delay_sec: "{{ au_delay_sec }}"
          correlation_id: "{{ au_correlation_id }}"
          original_emitted_at: "{{ au_original_emitted_at }}"
          schema_version: 1
      when: au_delay_sec is defined
      delegate_to: localhost

    - name: Debug  after-upgrade plan (routing + delay + version)
      ansible.builtin.debug:
        msg:
          - "routing_key={{ afterupgrade_routing_key }}"
          - "x-delay(ms)={{ (au_delay_sec | int) * 1000 }}"
          - "target_version={{ afterupgrade_payload.target_version | default('NA') }}"
      when: afterupgrade_payload is defined
      delegate_to: localhost

    - name: Debug  after-upgrade payload JSON (exactly what will be sent)
      delegate_to: localhost
      ansible.builtin.debug:
        msg: "{{ afterupgrade_payload | to_json }}"
      when: afterupgrade_payload is defined

    - name: Publish delayed after-upgrade check (headers.x-delay) to holding exchange
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
            headers:
              x-delay: "{{ (au_delay_sec | int) * 1000 }}"
          routing_key: "{{ afterupgrade_routing_key }}"
          payload: "{{ afterupgrade_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_afterupgrade_resp
      changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
      failed_when: >
        (rmq_afterupgrade_resp.status != 200) or
        (rmq_afterupgrade_resp.json is not defined)
      when: afterupgrade_payload is defined
      delegate_to: localhost

    # ---------------------------- Final operator summary (one-liners) ----------------------------
    - name: Summary key outcomes (one-liners)
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "dev2_conn_method={{ dev2_conn_method }}"
          - "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
          - "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
          - "dev2_passfile_used={{ dev2_passfile_used }}"
          - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
          - "bootenv_filename={{ bootenv_filename }}"
          - "bootenv_sha256={{ bootenv_sha256 }}"
          - "local_bootenv_present={{ local_img.stat.exists | default(false) }}"
          - "local_sha256={{ (local_sha256.stdout | default('NA')) | trim if (local_sha256 is defined) else 'NA' }}"
          - "dev2_sha256_before={{ (dev2_sha256_before.stdout | default('NA')) | trim if (dev2_sha256_before is defined) else 'NA' }}"
          - "dev2_sha256_after={{ (dev2_sha256_after.stdout | default('NA')) | trim if (dev2_sha256_after is defined) else 'NA' }}"
          - "bootenv_write_success={{ _bootenv_write_success | default(false) }}"
          - "reboot_requested={{ reboot_requested | default(false) }}"
          - "reboot_delay_seconds={{ reboot_delay_seconds if (reboot_requested | default(false)) else 'NA' }}"
          - "reboot_delay_minutes={{ reboot_delay_minutes if (reboot_requested | default(false)) else 'NA' }}"
          - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
          - "prep_blocked={{ _prep_blocked | default(false) }}"
          - "blocked={{ _blocked | default(false) }}"
          - "journal={{ (_journal | default([])) | join(' || ') }}"

  post_tasks:
    - name: Cleanup (always)
      block:
        - ansible.builtin.debug:
            msg: "Entering cleanup block"
          changed_when: false
          delegate_to: localhost
      always:
        - name: Close SSH ControlMaster (best-effort)
          delegate_to: localhost
          ansible.builtin.shell: |
            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
          changed_when: false
          ignore_errors: true

        - name: Remove tunnel control dir (best-effort)
          delegate_to: localhost
          ansible.builtin.file:
            path: "{{ _ctrl_dir | default('/tmp/none') }}"
            state: absent
          ignore_errors: true

        - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
          ansible.builtin.raw: >
            {{ pathprefix }}
            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
          register: del_ip
          changed_when: del_ip.rc == 0
          failed_when: >
            del_ip.rc != 0
            and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
            and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

    - name: Debug  temp IP removal result
      ansible.builtin.debug:
        msg:
          - "del_ip.rc={{ del_ip.rc | default('') }}"
          - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
          - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
      when: del_ip is defined