0920

files/ansible-playbooks/sot-updater.yml

@@ -0,0 +1,323 @@
+---
+# updater.yml — read Dev1 & Dev2 firmware versions and publish to controls → netbox-reporter
+# Run: nbplay updater.yml <device>
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Busybox-safe PATH prefix for raw calls on DEV1 (as in update-rebootin222)
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # DEV1 credentials (as in update-indoor)
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    # Tunnel target DEV2 behind DEV1 (as in update-indoor)
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+
+    # Password files sequence (as in update-indoor: try basicpass then basicpass2)
+    # NOTE: The selection below mirrors the exact “try #1 → select → try #2 → select → NONE” flow from update-indoor.
+    # Do not alter ordering.
+    # (Values are file names as used in your repo/environment.)
+    # We do not loop; we replicate the same task structure.
+    # The read of firmux uses whichever got selected.
+    # — Pavel’s rule: on-device commands remain immutable; we’re only orchestrating controller-side steps here.
+    
+    # Rabbit journaling (mirrors update-indoor / rebootin222)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ----------------------------- DEV1: read current banner (exactly like update-rebootin222) -----------------------------
+    - name: Dev1 | Read first /etc/banner line containing 'rev'
+      ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      changed_when: false
+
+    - name: Dev1 | Normalize '... rev NNNN' → '...-rNNNN'
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev1_fw_norm: >-
+          {{
+            (dev1_banner.stdout | default('') | trim)
+            | regex_replace('\\s*[Rr][Ee][Vv]\\.??\\s*([0-9]+)\\s*$', '-r\\1')
+          }}
+
+    - name: Publish → controls | custom_field_set fw_version (Dev1)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_norm | default('unknown'))
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------------- TUNNEL PREP (update-indoor blocks copied) -----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: add_ip.rc != 0
+                   and ('File exists' not in (add_ip.stdout | default('')))
+                   and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Pick a free local TCP port for the tunnel (controller)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+            for i in $(seq 1 25); do
+                p="$(shuf -i 20000-39999 -n 1)"
+                if command -v ss >/dev/null 2>&1; then
+                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                        echo "$p"; return 0
+                    fi
+                else
+                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                        echo "$p"; return 0
+                    fi
+                fi
+            done
+            return 1
+        }
+        pick
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+
+    - name: Create ControlMaster socket dir (mktemp)
+      delegate_to: localhost
+      set_fact:
+        _ctrl_dir: "{{ lookup('pipe', 'mktemp -d') }}"
+
+    - name: Compose ControlMaster socket path
+      delegate_to: localhost
+      set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local → DEV2:22 via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+            -o PreferredAuthentications=password -o PubkeyAuthentication=no \
+            -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30 \
+            -M -S "{{ _ctrl_sock }}" \
+            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+            "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel
+      delegate_to: localhost
+      ansible.builtin.shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    # ----------------------------- DEV2 AUTH PICK (exact task sequence from update-indoor) -----------------------------
+    - name: Try DEV2 login with 'basicpass' (root)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass' if previous login succeeded
+      when: dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login with 'basicpass2' (only if first failed)
+      when: dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass2' if previous login succeeded
+      when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Mark DEV2 auth as NONE if both attempts failed
+      when: dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    # ----------------------------- DEV2 firmware read (update-indoor firmux primary check) -----------------------------
+    - name: Dev2 | Read /usr/lib/release/firmux (if present)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      args: { executable: /bin/bash }
+      register: dev2_firmux
+      changed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Fallback to banner 'rev' if firmux not available
+      when: dev2_passfile_used != "NONE" and ((dev2_firmux.stdout | default('') | trim) | length == 0)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "grep -i rev /etc/banner 2>/dev/null | head -n1 || true"
+      args: { executable: /bin/bash }
+      register: dev2_banner
+      changed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Choose raw string (firmux preferred, else banner)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        indoor_fw_raw: >-
+          {{
+            (dev2_firmux.stdout | default('') | trim)
+            if ((dev2_firmux.stdout | default('') | trim) | length > 0)
+            else (dev2_banner.stdout | default('') | trim)
+          }}
+
+    - name: Dev2 | Normalize to X.Y.Z-rNNNN (convert trailing 'rev NNNN' → '-rNNNN')
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        indoor_fw_norm: >-
+          {{
+            ((indoor_fw_raw | default('') | trim | lower) is search('-r[0-9]+$'))
+            | ternary(
+                (indoor_fw_raw | default('') | trim),
+                ((indoor_fw_raw | default('') | trim) | regex_replace('\\s*[Rr][Ee][Vv]\\.??\\s*([0-9]+)\\s*$', '-r\\1'))
+              )
+          }}
+
+    - name: Publish → controls | custom_field_set indoor_fwver (Dev2)
+      when:
+        - dev2_passfile_used != "NONE"
+        - indoor_fw_norm is defined
+        - (indoor_fw_norm | length) > 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': indoor_fw_norm
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: Close SSH ControlMaster (best-effort)
+      delegate_to: localhost
+      ansible.builtin.shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: del_ip.rc != 0
+                   and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+                   and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/update-indoor.yml

@@ -1,5 +1,5 @@
 ---
-# update-secondline.yml
+# update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
   gather_facts: no
@@ -27,7 +27,7 @@
       - "basicpass2"
 
     # SSH options used from controller
-    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
 
     # Image to stage on DEV2 (we validate first; the actual write happens later)
     image_filename: "fox200-2.2.1-r6801.bin"
@@ -50,7 +50,6 @@
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
     afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
 
-    
   pre_tasks:
     # ------------------------------- Hostname sanity DEV1 -------------------------------
     - name: Read DEV1 hostname (busybox-safe)
@@ -165,10 +164,12 @@
       register: pick_port
       changed_when: false
 
+    # (moved up) Stop immediately if no free local port was found
     - name: Stop if no free local port was found
       ansible.builtin.meta: end_host
       when: (pick_port.stdout | trim | length) == 0
 
+    # (moved up) Set chosen port and control socket path
     - name: Record chosen local port and create control dir for SSH ControlMaster
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -180,15 +181,34 @@
       ansible.builtin.set_fact:
         _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
 
-    - name: Debug  tunnel parameters (controller side)
+    # (moved down) Now it’s safe to reference _local_port/_ctrl_sock
+    - name: Debug picked local port (controller)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
-          - "local_port={{ _local_port }}"
+          - "picked_local_port={{ _local_port }}"
-          - "ctrl_dir={{ _ctrl_dir }}"
           - "ctrl_sock={{ _ctrl_sock }}"
+          - "dev1_host={{ ansible_host | default(inventory_hostname) }}"
+          - "dev2_target={{ dev2_host }}:{{ dev2_port }}"
 
-    - name: Refresh ARP on DEV1’s LAN (send unsolicited ARP from temporary IP)
+    - name: Show current listeners on picked port (ss/lsof)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p'
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+      register: port_listeners_before
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (before starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "listeners_before:\n{{ (port_listeners_before.stdout | default('')) | trim }}"
+
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
       ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
 
     # ---------------------------- Start SSH local forward via DEV1 ----------------------------
@@ -207,6 +227,35 @@
       register: start_tunnel
       changed_when: true
 
+    - name: Debug ControlMaster start result (rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "start_tunnel.rc={{ start_tunnel.rc | default('NA') }}"
+          - "start_tunnel.stdout={{ (start_tunnel.stdout | default('')) | trim }}"
+          - "start_tunnel.stderr={{ (start_tunnel.stderr | default('')) | trim }}"
+
+    - name: Show who is listening now on the local port (post-start)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: port_listeners_after
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (after starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ (port_listeners_after.stdout | default('')) | trim }}"
+
     - name: Wait a moment for tunnel to settle
       delegate_to: localhost
       ansible.builtin.wait_for:
@@ -229,6 +278,14 @@
           - "tunnel_check.rc={{ tun_check.rc }}"
           - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
 
+    - name: Debug  ControlMaster check (full rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tun_check.rc={{ tun_check.rc | default('NA') }}"
+          - "tun_check.stdout={{ (tun_check.stdout | default('')) | trim }}"
+          - "tun_check.stderr={{ (tun_check.stderr | default('')) | trim }}"
+
     # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
     - name: Probe TCP reachability to DEV2 through the tunnel (nc)
       delegate_to: localhost
@@ -266,6 +323,33 @@
           - "{{ (dev2_ls.stdout | default('')) | trim }}"
           - "{{ (dev2_ls.stderr | default('')) | trim }}"
 
+    - name: Refresh ARP 2 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
+    # ---------------------------- Single banner probe (kept) ----------------------------
+    - name: Probe SSH banner through tunnel (pre-auth, quick)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        ssh -p "$PORT" \
+          -o PreferredAuthentications=none \
+          -o PubkeyAuthentication=no \
+          -o KbdInteractiveAuthentication=no \
+          -o PasswordAuthentication=no \
+          -o NumberOfPasswordPrompts=0 \
+          -o ConnectTimeout=5 \
+          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          -vvv root@127.0.0.1 true 2>&1 || true
+      register: tunnel_banner_probe
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  SSH preauth probe (first 40 lines)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "{{ (tunnel_banner_probe.stdout | default('') | split('\n'))[:40] | join('\n') }}"
+
     # ---------------------------- Pick DEV2 password for root ----------------------------
     - name: Try DEV2 login with 'basicpass' (root)
       delegate_to: localhost
@@ -276,12 +360,36 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass rc={{ dev2_try_basicpass.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass.stdout | default('')) | trim }}"
+
     - name: Select 'basicpass' if previous login succeeded
       when: dev2_try_basicpass.rc == 0
       delegate_to: localhost
@@ -299,12 +407,37 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass2
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass2 try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass2
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass2)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass2 rc={{ dev2_try_basicpass2.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass2.stdout | default('')) | trim }}"
+      when: dev2_try_basicpass2 is defined
+
     - name: Select 'basicpass2' if second login succeeded
       when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
       delegate_to: localhost
@@ -337,7 +470,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
       args:
@@ -456,6 +589,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ---------------------------- DEV2 version firmux primary check ----------------------------
     - name: Read DEV2 /usr/lib/release/firmux (if present)
       when: dev2_passfile_used != "NONE"
@@ -467,7 +603,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /usr/lib/release/firmux 2>/dev/null || true"
       args:
@@ -628,6 +764,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
 
+    - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # fw_printenv health before upload (soft-fail)
     - name: Read fw_printenv size (line count) on DEV2 (soft health)
       when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
@@ -663,7 +802,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
       args:
@@ -695,7 +834,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
       args:
@@ -746,7 +885,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "update -c '{{ dev2_image_path }}' 2>&1 || true"
       args:
@@ -804,6 +943,9 @@
       when: journal_indoor_aborted is defined
       delegate_to: localhost
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
     - name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
       when: not (_blocked | default(false))
@@ -981,8 +1123,6 @@
           register: dev2_marker_write
           changed_when: true
           ignore_errors: true
-   
-    
 
     # ---------------------------- Reboot scheduling (normalized) ----------------------------
     - name: Schedule DEV2 reboot after computed delay (seconds)

files/rabbit-client.sh

@@ -116,8 +116,8 @@ dispatch_task() {
     # now, today 01:00, tomorrow 01:00 (local time)
     local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
     now_s="$(date +%s)"
-    today1_s="$(date -d 'today 01:00' +%s)"
+    today1_s="$(date -d 'today 00:00' +%s)"
-    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 00:00' +%s)"
     if (( now_s < today1_s )); then
       next1_s="$today1_s"
     else
@@ -127,7 +127,7 @@ dispatch_task() {
     # Ceil hours so current minutes are preserved as in your examples
     ceil_h=$(( (diff_s + 3599) / 3600 ))
     rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
-    total_h=$(( ceil_h + rnd ))
+    total_h=$(( ceil_h + rnd - 1 ))
 
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"