0406

files/ansible-playbooks/sot-updater.yml

@@ -0,0 +1,596 @@
+---
+# sot-updater.yml — Read Dev1 & Dev2 firmware and publish to controls → netbox-reporter
+# Run: nbplay sot-updater.yml <device>
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Busybox-safe PATH prefix for *all* raw calls on DEV1
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # DEV1 credentials (controller-side)
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    # DEV2 behind DEV1
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    # DEV1 side addressing to reach DEV2
+    dev2_side_ip: "192.168.1.11/24"     # IP we add/remove on DEV1
+    dev1_iface: "br-wan"                # DEV1 interface where IP is added
+    arping_iface: "eth0"                # iface used for ARP refreshes
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    # Controller SSH common opts
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    # Rabbit (controls)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ───────────────────────── Progress: start ─────────────────────────
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ---------------------------- Temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists'; soft-fail)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: false
+      ignore_errors: true
+
+    - name: Soft-report if temp IP add failed (not just 'File exists')
+      when: add_ip.rc is defined and add_ip.rc != 0 and
+            ('File exists' not in (add_ip.stdout | default(''))) and
+            ('File exists' not in (add_ip.stderr | default('')))
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'temp_ip_add_failed',
+            'task_add1': 'rc=' ~ (add_ip.rc | string),
+            'task_add2': (add_ip.stderr | default(add_ip.stdout) | trim)[:512]
+          } | to_json }}"
+          payload_encoding: "string"
+
+    - name: Debug temp IP add
+      debug:
+        msg:
+          - "rc={{ add_ip.rc | default('') }}"
+          - "stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- AIRPING #1 (early refresh) ----------------------------
+    - name: Refresh ARP #1 on DEV1 LAN (unsolicited)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      register: arping1
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Local tunnel prep ----------------------------
+    - name: Pick a free local TCP port for the tunnel (controller)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      register: pick_port
+      changed_when: false
+
+    # ***** FIXED: split mktemp + sock path *****
+    - name: Record chosen local port
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+
+    - name: Create ControlMaster socket dir (mktemp -d)
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      register: mktemp_dir
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set _ctrl_dir fact (from mktemp) or leave empty
+      delegate_to: localhost
+      set_fact:
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+
+    - name: Journal if mktemp failed (no ctrl_dir)
+      when: _ctrl_dir | length == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'mktemp_dir_failed',
+            'task_add1': (mktemp_dir.stderr | default(mktemp_dir.stdout) | trim)[:256]
+          } | to_json }}"
+          payload_encoding: "string"
+
+    - name: Compose ControlMaster socket path (only if we have a dir)
+      when: _ctrl_dir | length > 0
+      delegate_to: localhost
+      set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    # ---------------------------- AIRPING #2 (before tunnel) ----------------------------
+    - name: Refresh ARP #2 on DEV1 LAN (pre-tunnel)
+      when: _local_port | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      register: arping2
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Start tunnel via DEV1 ----------------------------
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local → DEV2:22 via DEV1
+      when: _local_port | length > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: _local_port | length > 0
+      delegate_to: localhost
+      ansible.builtin.shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok fact
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ (_local_port | length > 0) and (nc_probe.rc is defined) and (nc_probe.rc == 0) }}"
+
+    - name: Journal if tunnel probe failed
+      when: not tunnel_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'tunnel_probe_failed',
+            'task_add1': 'port=' ~ (_local_port | default('n/a'))
+          } | to_json }}"
+          payload_encoding: "string"
+
+    # ───────────────────────── DEV1: banner → fw_version ─────────────────────────
+    - name: Dev1 | Probe banner via SSH from controller (classic extraction)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    # ***** FIXED: stepwise, flag-safe normalization for DEV1 *****
+    - name: Dev1 | Debug raw banner
+      when: dev1_banner is defined
+      delegate_to: localhost
+      debug:
+        msg:
+          - "DEV1 banner raw: {{ (dev1_banner.stdout | default('') | trim) | regex_replace('\n',' ') }}"
+
+    - name: Dev1 | Strip trailing ' | ...'
+      delegate_to: localhost
+      set_fact:
+        dev1_no_ts: "{{ (dev1_banner.stdout | default('') | trim) | regex_replace('\\|.*$', '') }}"
+
+    - name: Dev1 | Convert 'rev YYYY' → '-rYYYY'
+      delegate_to: localhost
+      set_fact:
+        dev1_rev_dash: "{{ dev1_no_ts | regex_replace('rev\\.?\\s*([0-9]+)\\s*$', '-r\\1', ignorecase=True) }}"
+
+    - name: Dev1 | Extract X.X.X-rYYYY core
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_rev_dash | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+') | default('', true) }}"
+
+    - name: Journal if DEV1 normalization failed (no core)
+      when: (dev1_fw_clean | default('') | length) == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'dev1_banner_parse_failed',
+            'task_add1': (dev1_banner.stdout | default('NO_STDOUT') | trim)[:256]
+          } | to_json }}"
+          payload_encoding: "string"
+
+    - name: Publish → controls | custom_field_set fw_version (Dev1)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean | length) > 0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      register: dev1_publish_resp
+      changed_when: false
+
+    # ───────────────────────── DEV2 auth pick ─────────────────────────
+    - name: Try DEV2 login with 'basicpass' (root)
+      when: tunnel_ok | default(false)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass' if previous login succeeded
+      when: tunnel_ok | default(false) and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+
+    - name: Try DEV2 login with 'basicpass2' (only if first failed)
+      when: tunnel_ok | default(false) and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass2
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass2' if previous login succeeded
+      when: tunnel_ok | default(false) and (dev2_passfile_used is not defined) and (dev2_try_basicpass2.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+
+    - name: Mark DEV2 auth as NONE if both attempts failed
+      when: tunnel_ok | default(false) and (dev2_passfile_used is not defined)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+
+    - name: Journal if DEV2 auth failed
+      when: tunnel_ok | default(false) and (dev2_passfile_used == 'NONE')
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'dev2_auth_failed'
+          } | to_json }}"
+          payload_encoding: "string"
+
+    # ---------------------------- AIRPING #3 (post-tunnel stabilizer) ----------------------------
+    - name: Refresh ARP #3 on DEV1 LAN (post-tunnel)
+      when: tunnel_ok | default(false)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      register: arping3
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    # ───────────────────────── DEV2: firmux → indoor_fwver ─────────────────────────
+    - name: Dev2 | Read /usr/lib/release/firmux (if authenticated)
+      when: tunnel_ok | default(false) and dev2_passfile_used is defined and dev2_passfile_used != 'NONE'
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      args: { executable: /bin/bash }
+      register: dev2_firmux
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Debug raw firmux
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      debug:
+        msg:
+          - "DEV2 firmux raw: {{ (dev2_firmux.stdout | default('') | trim) | regex_replace('\n',' | ') }}"
+
+    - name: Dev2 | Build firmux lines list
+      delegate_to: localhost
+      set_fact:
+        _firmux_lines: "{{ (dev2_firmux.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}"
+
+    - name: Dev2 | Choose firmux line (prefer containing 'rev NNNN')
+      delegate_to: localhost
+      set_fact:
+        dev2_line: >-
+          {{
+            (_firmux_lines | select('match','(?i).*\\brev\\s*\\d+.*') | list | first)
+              | default((_firmux_lines | select('truthy') | list | first) | default(''))
+          }}
+
+    # ***** FIXED: ensure dev2_line_clean is defined before use, stepwise extraction *****
+    - name: Dev2 | Strip trailing pipes and whitespace
+      delegate_to: localhost
+      set_fact:
+        dev2_line_clean: "{{ (dev2_line | default('') | trim) | regex_replace('\\|.*$', '') }}"
+
+    - name: Dev2 | Extract version (X.X.X) and rev number
+      delegate_to: localhost
+      set_fact:
+        dev2_ver: "{{ dev2_line_clean | regex_search('[0-9]+\\.[0-9]+\\.[0-9]+') | default('', true) }}"
+        dev2_rev_token: "{{ dev2_line_clean | regex_search('(?i)\\brev\\.?\\s*[0-9]+') | default('', true) }}"
+        dev2_rev_num: "{{ (dev2_rev_token | regex_replace('(?i).*\\brev\\.?\\s*', '')) if (dev2_rev_token | length) > 0 else '' }}"
+
+    - name: Dev2 | Build EXACT 'X.X.X rev YYYY' (lowercase 'rev', single space)
+      delegate_to: localhost
+      set_fact:
+        indoor_fw_norm: "{{ (dev2_ver ~ ' rev ' ~ dev2_rev_num) if (dev2_ver|length>0 and dev2_rev_num|length>0) else '' }}"
+
+    - name: Journal if DEV2 normalization failed (no version or rev)
+      when: tunnel_ok | default(false) and (indoor_fw_norm | default('') | length) == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'dev2_firmux_parse_failed',
+            'task_add1': (dev2_firmux.stdout | default('NO_STDOUT') | trim)[:256]
+          } | to_json }}"
+          payload_encoding: "string"
+
+    - name: Publish → controls | custom_field_set indoor_fwver (Dev2)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (
+              indoor_fw_norm if (tunnel_ok | default(false)) and (indoor_fw_norm | default('') | length) > 0
+              else ('unreachable' if not (tunnel_ok | default(false)) else 'unavailable')
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      register: dev2_publish_resp
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Progress | Cleanup start (close tunnel & remove temp IP)"
+      debug:
+        msg: "Closing ControlMaster and removing temporary IP"
+
+    # ---------------------------- AIRPING #4 (pre-cleanup) ----------------------------
+    - name: Refresh ARP #4 on DEV1 LAN (pre-cleanup)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      register: arping4
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster (best-effort)
+      delegate_to: localhost
+      ansible.builtin.shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address'; soft-fail)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: false
+      ignore_errors: true
+
+    - name: Journal if temp IP delete failed (not just 'Cannot assign requested address')
+      when: del_ip.rc is defined and del_ip.rc != 0 and
+            ('Cannot assign requested address' not in (del_ip.stdout | default(''))) and
+            ('Cannot assign requested address' not in (del_ip.stderr | default('')))
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_entry',
+            'task_result': 'temp_ip_del_failed',
+            'task_add1': 'rc=' ~ (del_ip.rc | string),
+            'task_add2': (del_ip.stderr | default(del_ip.stdout) | trim)[:512]
+          } | to_json }}"
+          payload_encoding: "string"
+
+    - name: "✅ Completed | Device processed (Dev1+Dev2)."
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE (report-only)"

files/ansible-playbooks/update-indoor.yml

@@ -1,5 +1,5 @@
 ---
-# update-secondline.yml
+# update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
   gather_facts: no
@@ -27,7 +27,7 @@
       - "basicpass2"
 
     # SSH options used from controller
-    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
 
     # Image to stage on DEV2 (we validate first; the actual write happens later)
     image_filename: "fox200-2.2.1-r6801.bin"
@@ -50,7 +50,6 @@
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
     afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
 
-    
   pre_tasks:
     # ------------------------------- Hostname sanity DEV1 -------------------------------
     - name: Read DEV1 hostname (busybox-safe)
@@ -165,10 +164,12 @@
       register: pick_port
       changed_when: false
 
+    # (moved up) Stop immediately if no free local port was found
     - name: Stop if no free local port was found
       ansible.builtin.meta: end_host
       when: (pick_port.stdout | trim | length) == 0
 
+    # (moved up) Set chosen port and control socket path
     - name: Record chosen local port and create control dir for SSH ControlMaster
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -180,15 +181,34 @@
       ansible.builtin.set_fact:
         _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
 
-    - name: Debug  tunnel parameters (controller side)
+    # (moved down) Now it’s safe to reference _local_port/_ctrl_sock
+    - name: Debug picked local port (controller)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
-          - "local_port={{ _local_port }}"
+          - "picked_local_port={{ _local_port }}"
-          - "ctrl_dir={{ _ctrl_dir }}"
           - "ctrl_sock={{ _ctrl_sock }}"
+          - "dev1_host={{ ansible_host | default(inventory_hostname) }}"
+          - "dev2_target={{ dev2_host }}:{{ dev2_port }}"
 
-    - name: Refresh ARP on DEV1’s LAN (send unsolicited ARP from temporary IP)
+    - name: Show current listeners on picked port (ss/lsof)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p'
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+      register: port_listeners_before
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (before starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "listeners_before:\n{{ (port_listeners_before.stdout | default('')) | trim }}"
+
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
       ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
 
     # ---------------------------- Start SSH local forward via DEV1 ----------------------------
@@ -207,6 +227,35 @@
       register: start_tunnel
       changed_when: true
 
+    - name: Debug ControlMaster start result (rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "start_tunnel.rc={{ start_tunnel.rc | default('NA') }}"
+          - "start_tunnel.stdout={{ (start_tunnel.stdout | default('')) | trim }}"
+          - "start_tunnel.stderr={{ (start_tunnel.stderr | default('')) | trim }}"
+
+    - name: Show who is listening now on the local port (post-start)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: port_listeners_after
+      changed_when: false
+      failed_when: false
+
+    - name: Debug listeners on picked port (after starting tunnel)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ (port_listeners_after.stdout | default('')) | trim }}"
+
     - name: Wait a moment for tunnel to settle
       delegate_to: localhost
       ansible.builtin.wait_for:
@@ -229,6 +278,14 @@
           - "tunnel_check.rc={{ tun_check.rc }}"
           - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
 
+    - name: Debug  ControlMaster check (full rc/stdout/stderr)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tun_check.rc={{ tun_check.rc | default('NA') }}"
+          - "tun_check.stdout={{ (tun_check.stdout | default('')) | trim }}"
+          - "tun_check.stderr={{ (tun_check.stderr | default('')) | trim }}"
+
     # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
     - name: Probe TCP reachability to DEV2 through the tunnel (nc)
       delegate_to: localhost
@@ -266,6 +323,33 @@
           - "{{ (dev2_ls.stdout | default('')) | trim }}"
           - "{{ (dev2_ls.stderr | default('')) | trim }}"
 
+    - name: Refresh ARP 2 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
+    # ---------------------------- Single banner probe (kept) ----------------------------
+    - name: Probe SSH banner through tunnel (pre-auth, quick)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        ssh -p "$PORT" \
+          -o PreferredAuthentications=none \
+          -o PubkeyAuthentication=no \
+          -o KbdInteractiveAuthentication=no \
+          -o PasswordAuthentication=no \
+          -o NumberOfPasswordPrompts=0 \
+          -o ConnectTimeout=5 \
+          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          -vvv root@127.0.0.1 true 2>&1 || true
+      register: tunnel_banner_probe
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  SSH preauth probe (first 40 lines)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "{{ (tunnel_banner_probe.stdout | default('') | split('\n'))[:40] | join('\n') }}"
+
     # ---------------------------- Pick DEV2 password for root ----------------------------
     - name: Try DEV2 login with 'basicpass' (root)
       delegate_to: localhost
@@ -276,12 +360,36 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass rc={{ dev2_try_basicpass.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass.stdout | default('')) | trim }}"
+
     - name: Select 'basicpass' if previous login succeeded
       when: dev2_try_basicpass.rc == 0
       delegate_to: localhost
@@ -299,12 +407,37 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
       register: dev2_try_basicpass2
       changed_when: false
       ignore_errors: true
 
+    - name: Snapshot listeners on local tunnel port (after basicpass2 try)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        P="{{ _local_port }}"
+        echo "== ss -ltnp on :${P} =="
+        { ss -ltnp 2>/dev/null || true; } | awk -v p=":${P}$" '$0 ~ p {print}'
+        echo "== lsof LISTEN on :${P} =="
+        { lsof -nP -iTCP:"${P}" -sTCP:LISTEN 2>/dev/null || true; }
+        echo "== ps/grep ControlMaster by ControlPath =="
+        ps -ef | grep -F " -S {{ _ctrl_sock }}" | grep -v grep || true
+      register: listeners_after_basicpass2
+      changed_when: false
+      failed_when: false
+
+    - name: Debug  auth try context (basicpass2)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "auth_try=basicpass2 rc={{ dev2_try_basicpass2.rc | default('NA') }}"
+          - "local_port={{ _local_port }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+          - "listeners:\n{{ (listeners_after_basicpass2.stdout | default('')) | trim }}"
+      when: dev2_try_basicpass2 is defined
+
     - name: Select 'basicpass2' if second login succeeded
       when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
       delegate_to: localhost
@@ -337,7 +470,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
       args:
@@ -456,6 +589,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ---------------------------- DEV2 version firmux primary check ----------------------------
     - name: Read DEV2 /usr/lib/release/firmux (if present)
       when: dev2_passfile_used != "NONE"
@@ -467,7 +603,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "cat /usr/lib/release/firmux 2>/dev/null || true"
       args:
@@ -628,6 +764,9 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
 
+    - name: Refresh ARP 3 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # fw_printenv health before upload (soft-fail)
     - name: Read fw_printenv size (line count) on DEV2 (soft health)
       when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
@@ -663,7 +802,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
       args:
@@ -695,7 +834,7 @@
         sshpass -f "{{ dev2_passfile_used }}" ssh \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
       args:
@@ -746,7 +885,7 @@
           -o AddressFamily=inet \
           -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
           -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
+          -o ConnectTimeout=30 \
           -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
           "update -c '{{ dev2_image_path }}' 2>&1 || true"
       args:
@@ -804,6 +943,9 @@
       when: journal_indoor_aborted is defined
       delegate_to: localhost
 
+    - name: Refresh ARP 1 on DEV1’s LAN (send unsolicited ARP from temporary IP)
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
     # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
     - name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
       when: not (_blocked | default(false))
@@ -981,8 +1123,6 @@
           register: dev2_marker_write
           changed_when: true
           ignore_errors: true
-   
-    
 
     # ---------------------------- Reboot scheduling (normalized) ----------------------------
     - name: Schedule DEV2 reboot after computed delay (seconds)

files/rabbit-client.sh

@@ -116,8 +116,8 @@ dispatch_task() {
     # now, today 01:00, tomorrow 01:00 (local time)
     local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
     now_s="$(date +%s)"
-    today1_s="$(date -d 'today 01:00' +%s)"
+    today1_s="$(date -d 'today 00:00' +%s)"
-    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 00:00' +%s)"
     if (( now_s < today1_s )); then
       next1_s="$today1_s"
     else
@@ -127,7 +127,7 @@ dispatch_task() {
     # Ceil hours so current minutes are preserved as in your examples
     ceil_h=$(( (diff_s + 3599) / 3600 ))
     rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
-    total_h=$(( ceil_h + rnd ))
+    total_h=$(( ceil_h + rnd - 1 ))
 
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"