0957

files/ansible-playbooks/sot-updater.yml

@@ -0,0 +1,371 @@
+---
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug result of adding temp IP to DEV1
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_mac is defined and dev2_mac | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug ARP action summary on DEV1
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- ARP refresh ----------------------------
+    - name: Refresh ARP #1
+      ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+
+    - name: Refresh ARP #1
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Local tunnel prep -----------------------
+    - name: Pick a free local TCP port for the tunnel
+      delegate_to: localhost
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Record chosen port and create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set facts for tunnel paths
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
+
+    # ----------------------- AIRPING #2 -----------------------
+    - name: Refresh ARP #2
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Start tunnel -----------------------
+    - name: Start SSH tunnel via DEV1
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- DEV1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
+
+    - name: Debug Dev1 normalized firmware
+      delegate_to: localhost
+      debug:
+        msg: "Dev1 fw_version  {{ dev1_fw_clean | default('N/A') }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- DEV2 auth -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok | default(false)
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set dev2_passfile_used
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
+
+    # ----------------------- AIRPING #3 -----------------------
+    - name: Refresh ARP #3
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+
+    # ----------------------- DEV2 firmux (simplified, literal) -----------------------
+    - name: Dev2 | Read /usr/lib/release/firmux
+      when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
+      delegate_to: localhost
+      shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      register: dev2_firmux
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
+      delegate_to: localhost
+      run_once: true
+      set_fact:
+        indoor_fw_norm: >-
+          {{
+            (
+              (
+                dev2_firmux.stdout | default('') | regex_replace('\r','')
+              ) | split('\n')
+              | map('trim')
+              | select('truthy')
+              | list
+            ) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
+              | list
+              | first
+              | default(
+                  ((dev2_firmux.stdout | default('') | regex_replace('\r','')) 
+                  | split('\n') 
+                  | map('trim') 
+                  | select('truthy') 
+                  | list 
+                  | first 
+                  | default(''))
+                )
+              | trim
+          }}
+
+    - name: Publish Dev2 indoor_fwver
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP #4
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster
+      delegate_to: localhost
+      shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir
+      delegate_to: localhost
+      file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE"

files/ansible-playbooks/update-indoor.yml

@@ -1,5 +1,5 @@
 ---
-# update-secondline.yml
+# update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
   gather_facts: no
@@ -27,7 +27,7 @@
       - "basicpass2"
 
     # SSH options used from controller
-    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
 
     # Image to stage on DEV2 (we validate first; the actual write happens later)
     image_filename: "fox200-2.2.1-r6801.bin"
@@ -50,7 +50,6 @@
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
     afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
 
-    
   pre_tasks:
     # ------------------------------- Hostname sanity DEV1 -------------------------------
     - name: Read DEV1 hostname (busybox-safe)
@@ -165,10 +164,12 @@
       register: pick_port
       changed_when: false
 
+    # (moved up) Stop immediately if no free local port was found
     - name: Stop if no free local port was found
       ansible.builtin.meta: end_host
       when: (pick_port.stdout | trim | length) == 0
 
+    # (moved up) Set chosen port and control socket path
     - name: Record chosen local port and create control dir for SSH ControlMaster
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -180,15 +181,34 @@
       ansible.builtin.set_fact:
         _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
 
-    - name: Debug  tunnel parameters (controller side)
+    # (moved down) Now it’s safe to reference _local_port/_ctrl_sock
+    - name: Debug picked local port (controller)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
-          - "local_port={{ _local_port }}"
+          - "picked_local_port={{ _local_port }}"
-          - "ctrl_dir={{ _ctrl_dir }}"
           - "ctrl_sock={{ _ctrl_sock }}"
+          - "dev1_host={{ ansible_host | default(inventory_hostname) }}"
+          - "dev2_target={{ dev2_host }}:{{ dev2_port }}"
 
-    - name: Refresh ARP on DEV1’s LAN (send unsolicited ARP from temporary IP)
+    - name: Show current listeners on picked port (ss/lsof)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p'
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+      register: port_listeners_before
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (before starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "listeners_before:\n{{ (port_listeners_before.stdout | default('')) | trim }}"
+
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
       ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
 
     # ---------------------------- Start SSH local forward via DEV1 ----------------------------
@@ -207,6 +227,35 @@
       register: start_tunnel
       changed_when: true
 
+    - name: Debug ControlMaster start result (rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "start_tunnel.rc={{ start_tunnel.rc | default('NA') }}"
+          - "start_tunnel.stdout={{ (start_tunnel.stdout | default('')) | trim }}"
+          - "start_tunnel.stderr={{ (start_tunnel.stderr | default('')) | trim }}"
+
+    - name: Show who is listening now on the local port (post-start)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: port_listeners_after
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (after starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ (port_listeners_after.stdout | default('')) | trim }}"
+
     - name: Wait a moment for tunnel to settle
       delegate_to: localhost
       ansible.builtin.wait_for:
@@ -229,6 +278,14 @@
           - "tunnel_check.rc={{ tun_check.rc }}"
           - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
 
+    - name: Debug  ControlMaster check (full rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tun_check.rc={{ tun_check.rc | default('NA') }}"
+          - "tun_check.stdout={{ (tun_check.stdout | default('')) | trim }}"
+          - "tun_check.stderr={{ (tun_check.stderr | default('')) | trim }}"
+
     # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
     - name: Probe TCP reachability to DEV2 through the tunnel (nc)
       delegate_to: localhost
@@ -266,6 +323,33 @@
           - "{{ (dev2_ls.stdout | default('')) | trim }}"
           - "{{ (dev2_ls.stderr | default('')) | trim }}"
 
+    - name: Refresh ARP 2 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
+    # ---------------------------- Single banner probe (kept) ----------------------------
+    - name: Probe SSH banner through tunnel (pre-auth, quick)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        ssh -p "$PORT" \
+          -o PreferredAuthentications=none \
+          -o PubkeyAuthentication=no \
+          -o KbdInteractiveAuthentication=no \
+          -o PasswordAuthentication=no \
+          -o NumberOfPasswordPrompts=0 \
+          -o ConnectTimeout=5 \
+          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          -vvv root@127.0.0.1 true 2>&1 || true
+      register: tunnel_banner_probe
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  SSH preauth probe (first 40 lines)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "{{ (tunnel_banner_probe.stdout | default('') | split('\n'))[:40] | join('\n') }}"
+
     # ---------------------------- Pick DEV2 password for root ----------------------------
     - name: Try DEV2 login with 'basicpass' (root)
       delegate_to: localhost
@@ -276,12 +360,36 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass rc={{ dev2_try_basicpass.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass.stdout | default('')) | trim }}"
+
     - name: Select 'basicpass' if previous login succeeded
       when: dev2_try_basicpass.rc == 0
       delegate_to: localhost
@@ -299,12 +407,37 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass2
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass2 try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass2
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass2)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass2 rc={{ dev2_try_basicpass2.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass2.stdout | default('')) | trim }}"
+      when: dev2_try_basicpass2 is defined
+
     - name: Select 'basicpass2' if second login succeeded
       when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
       delegate_to: localhost
@@ -337,7 +470,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
       args:
@@ -456,6 +589,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ---------------------------- DEV2 version firmux primary check ----------------------------
     - name: Read DEV2 /usr/lib/release/firmux (if present)
       when: dev2_passfile_used != "NONE"
@@ -467,7 +603,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /usr/lib/release/firmux 2>/dev/null || true"
       args:
@@ -628,6 +764,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
 
+    - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # fw_printenv health before upload (soft-fail)
     - name: Read fw_printenv size (line count) on DEV2 (soft health)
       when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
@@ -663,7 +802,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
       args:
@@ -695,7 +834,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
       args:
@@ -746,7 +885,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "update -c '{{ dev2_image_path }}' 2>&1 || true"
       args:
@@ -804,6 +943,9 @@
       when: journal_indoor_aborted is defined
       delegate_to: localhost
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
     - name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
       when: not (_blocked | default(false))
@@ -981,8 +1123,6 @@
           register: dev2_marker_write
           changed_when: true
           ignore_errors: true
-   
-    
 
     # ---------------------------- Reboot scheduling (normalized) ----------------------------
     - name: Schedule DEV2 reboot after computed delay (seconds)

files/rabbit-client.sh

@@ -116,8 +116,8 @@ dispatch_task() {
     # now, today 01:00, tomorrow 01:00 (local time)
     local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
     now_s="$(date +%s)"
-    today1_s="$(date -d 'today 01:00' +%s)"
+    today1_s="$(date -d 'today 00:00' +%s)"
-    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 00:00' +%s)"
     if (( now_s < today1_s )); then
       next1_s="$today1_s"
     else
@@ -127,7 +127,7 @@ dispatch_task() {
     # Ceil hours so current minutes are preserved as in your examples
     ceil_h=$(( (diff_s + 3599) / 3600 ))
     rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
-    total_h=$(( ceil_h + rnd ))
+    total_h=$(( ceil_h + rnd - 1 ))
 
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"