first commit

files/ansible-playbooks/afterupgrade_check.yml

@@ -0,0 +1,355 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_check.yml-from_chat

@@ -0,0 +1,355 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          expiration: "{{ (next_delay_ms | int) | string }}"
+          correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_check.yml-from_container

@@ -0,0 +1,432 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}'
+            Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag ---
+
+    - name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801")
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        indoor_fwver_norm: >-
+          {{
+            ( (dev2_fwver.stdout | default('') | trim)
+              | regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1')
+              | regex_replace('\\s+', ' ')
+              | trim
+            )
+          }}
+
+    - name: Publish NetBox custom field indoor_fwver to control queue
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fwver_norm | default(''))
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cf_indoor_fwver
+      changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool)
+
+    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_remove_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_remove"
+          task_result: "indoor-restart-scheduled"
+
+    - name: Publish indoor-restart-scheduled tag removal
+      when: tag_remove_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_remove_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_remove_sched
+      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
+
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host
+

files/ansible-playbooks/afterupgrade_check.yml-screewed

@@ -0,0 +1,359 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}'
+            Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_indoor_check.yml

@@ -0,0 +1,744 @@
+---
+- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # ---------------- BusyBox-safe path prefix ----------------
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # ---------------- DEV1 (outer device) ----------------
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    dev1_iface: "br-wan"
+
+    # ---------------- DEV2 (indoor behind DEV1) ----------------
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ---- Normalize metadata (from delayed message) ----
+    - name: Normalize after-upgrade metadata
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        target_version_full: "{{ target_version | default('') }}"
+
+    # (NEW) Show what we received from the scheduler (for easy troubleshooting)
+    - name: Debug received scheduler metadata
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
+    - name: Check if TCP/22 on DEV1 is reachable
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # ---- If TCP down: journal + schedule next try or give up ----
+    - name: Build failure journal (TCP unreachable) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP 22 unreachable. Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (TCP unreachable)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    - name: Compute next-attempt delay (10 minutes) and counters
+      when: (_needs_retry | default(false)) | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: 600000
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path=TCP.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    - name: Build delayed after-upgrade payload for next attempt (10m)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after scheduling next attempt
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host
+
+
+    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname
+      when: nc_probe.rc == 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 hostname mismatch
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+    # ---------------- Step 2: Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 (ignore if exists)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0 and
+        ('File exists' not in (add_ip.stdout | default(''))) and
+        ('File exists' not in (add_ip.stderr | default('')))
+
+    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        for i in $(seq 1 25); do
+          p="$(shuf -i 20000-39999 -n1)"
+          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
+        done
+        exit 1
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control dir
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Start SSH ControlMaster tunnel via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
+          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
+    - name: Small delay for tunnel to settle
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        timeout: 1
+      changed_when: false
+
+    - name: Verify tunnel master running (ssh -O check)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
+      register: tun_check
+      changed_when: false
+
+    - name: Debug tunnel check
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tunnel_check.rc={{ tun_check.rc }}"
+          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
+
+    - name: Sanity confirm tunnel TCP reachability to DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug reachability result
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "nc.rc={{ nc_probe.rc }}"
+          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
+          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
+
+    - name: Stop if tunnel TCP check failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    # ---------------- Step 4: Determine working password for DEV2 ----------------
+    - name: Try both passfiles for DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
+            echo "$f"; exit 0;
+          fi
+        done
+        echo NONE
+      register: dev2_passfile_try
+      changed_when: false
+
+    - name: Save selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
+      changed_when: false
+
+    - name: Debug selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "dev2_passfile_used={{ dev2_passfile_used }}"
+
+    - name: Stop if no valid passfile found
+      ansible.builtin.meta: end_host
+      when: dev2_passfile_used == "NONE"
+
+    # ---------------- Step 5: Read firmware version on DEV2 ----------------
+    - name: Read firmware version from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
+      register: dev2_fwver
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show firmware version readout
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    # ---------------- Retry metadata + success evaluation (read OK) ----------------
+    - name: Normalize retry metadata for indoor checker
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+
+    - name: Check if firmware read succeeded (read_ok)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
+                    and ((dev2_fwver.stdout | default('') | trim) | length > 0)
+                    and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
+
+    # ===================== (NEW) Unconditional normalization + comparison =====================
+    # Compute expected_norm from target_version_full. If the full filename-like string is sent,
+    # we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
+
+    - name: Normalize expected target string (step 1: compute components)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
+        expected_norm_core: >-
+          {{
+            (target_version_full | default('') |
+              regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1'))
+            | default('', true)
+          }}
+
+    - name: Normalize expected target string (step 2: choose core if present)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+
+    # Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801"
+    - name: Normalize banner/firmux string from DEV2
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
+        banner_norm: >-
+          {{
+            (banner_raw | lower is search('-r[0-9]+$'))
+            | ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')))
+          }}
+
+    - name: Evaluate version match (normalized equality or contains)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        version_match: >-
+          {{
+            (expected_norm | length > 0)
+            and (
+              (banner_norm == expected_norm)
+              or (banner_norm is search(expected_norm))
+              or (expected_norm is search(banner_norm))
+            )
+          }}
+
+    - name: Debug compare snapshot (expected vs actual normalized)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm='{{ expected_norm }}'"
+          - "banner_norm='{{ banner_norm }}'"
+          - "version_match={{ version_match | default(false) }}"
+
+    # ===================== Journaling/Tagging paths =====================
+    # SUCCESS: read_ok AND version_match
+    - name: Build success journal payload
+      when: (read_ok | bool) and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ banner_raw }}' Target='{{ expected_norm }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_success
+      changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
+
+    - name: Build payload to add indoor-update-success tag (DEV1)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "indoor-update-success"
+
+    - name: Publish indoor-update-success tag
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add_success
+      changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
+
+    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_remove_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_remove"
+          task_result: "indoor-restart-scheduled"
+
+    - name: Publish indoor-restart-scheduled tag removal
+      when: tag_remove_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_remove_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_remove_sched
+      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
+
+    # --- Normalize firmware string and set custom field on success ---
+    - name: Capture raw firmware banner from DEV2 (for normalization)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_banner_raw: "{{ banner_raw }}"
+
+    - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
+      when:
+        - journal_success_payload is defined
+        - (fw_banner_raw | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_norm: >-
+          {{
+            fw_banner_raw
+            if (fw_banner_raw | lower is search('-r[0-9]+$'))
+            else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
+          }}
+
+    - name: Debug normalized firmware (indoor_fwver)
+      when: fw_norm is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
+
+    - name: Publish custom_field_set indoor_fwver
+      when:
+        - fw_norm is defined
+        - (fw_norm | length) > 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': fw_norm
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_customfield_fw
+      changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
+
+    # MISMATCH path: firmware readable but does NOT match expected target
+    - name: Build mismatch journal payload
+      when: (read_ok | bool) and (not (version_match | bool))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    - name: Stop host after mismatch evaluation
+      when: journal_mismatch_payload is defined
+      ansible.builtin.meta: end_host
+
+    # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
+    - name: Build failure journal payload (indoor firmware read failed)
+      when: not (read_ok | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish failure journal to control queue
+      when: journal_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_fail
+      changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
+
+    - name: Compute retry parameters (10 minutes)
+      when: not (read_ok | bool)
+      ansible.builtin.set_fact:
+        next_attempt: "{{ (attempt | int) + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: "{{ 600000 }}"
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_gaveup
+      changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
+
+    # Only schedule next attempt if we still have budget left
+    - name: Build delayed payload for next indoor attempt (10 min)
+      when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id | default('') }}"
+          original_emitted_at: "{{ original_emitted_at | default('') }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id | default('') }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+  post_tasks:
+    - name: Cleanup tunnel and temp IP
+      block:
+        - ansible.builtin.debug:
+            msg: "Cleaning up tunnel + temp IP"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP from DEV1
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          failed_when: false
+          changed_when: false

files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries

@@ -0,0 +1,201 @@
+---
+- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # ---------------- BusyBox-safe path prefix ----------------
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # ---------------- DEV1 (outer device) ----------------
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    dev1_iface: "br-wan"
+
+    # ---------------- DEV2 (indoor behind DEV1) ----------------
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 hostname mismatch
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+    # ---------------- Step 2: Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 (ignore if exists)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0 and
+        ('File exists' not in (add_ip.stdout | default(''))) and
+        ('File exists' not in (add_ip.stderr | default('')))
+
+    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        for i in $(seq 1 25); do
+          p="$(shuf -i 20000-39999 -n1)"
+          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
+        done
+        exit 1
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control dir
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+
+    - name: Start SSH ControlMaster tunnel via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
+          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    # ---------------- Step 4: Determine working password for DEV2 ----------------
+    - name: Try both passfiles for DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
+            echo "$f"; exit 0;
+          fi
+        done
+        echo NONE
+      register: dev2_passfile_try
+      changed_when: false
+
+    - name: Save selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
+      changed_when: false
+
+    - name: Stop if no valid passfile found
+      ansible.builtin.meta: end_host
+      when: dev2_passfile_used == "NONE"
+
+    # ---------------- Step 5: Read firmware version on DEV2 ----------------
+    - name: Read firmware version from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
+      register: dev2_fwver
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show firmware version readout
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    # ---------------- Step 6: Publish result journal ----------------
+    - name: Build journal payload
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
+            tunnel_port={{ _local_port }}
+
+    - name: Publish journal to control queue
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_fwver
+      changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool)
+
+  post_tasks:
+    - name: Cleanup tunnel and temp IP
+      block:
+        - ansible.builtin.debug:
+            msg: "Cleaning up tunnel + temp IP"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP from DEV1
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          failed_when: false
+          changed_when: false

files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors

@@ -0,0 +1,644 @@
+---
+- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # ---------------- BusyBox-safe path prefix ----------------
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # ---------------- DEV1 (outer device) ----------------
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    dev1_iface: "br-wan"
+
+    # ---------------- DEV2 (indoor behind DEV1) ----------------
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ---- Normalize metadata (from delayed message) ----
+    - name: Normalize after-upgrade metadata
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        target_version_full: "{{ target_version | default('') }}"
+
+    # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
+    - name: Check if TCP/22 on DEV1 is reachable
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # ---- If TCP down: journal + schedule next try or give up ----
+    - name: Build failure journal (TCP unreachable) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP 22 unreachable. Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (TCP unreachable)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    - name: Compute next-attempt delay (10 minutes) and counters
+      when: (_needs_retry | default(false)) | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: 600000
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path=TCP.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    - name: Build delayed after-upgrade payload for next attempt (10m)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after scheduling next attempt
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host
+
+
+    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname
+      when: nc_probe.rc == 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 hostname mismatch
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+    # ---------------- Step 2: Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 (ignore if exists)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0 and
+        ('File exists' not in (add_ip.stdout | default(''))) and
+        ('File exists' not in (add_ip.stderr | default('')))
+
+    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        for i in $(seq 1 25); do
+          p="$(shuf -i 20000-39999 -n1)"
+          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
+        done
+        exit 1
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control dir
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Start SSH ControlMaster tunnel via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
+          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
+    - name: Small delay for tunnel to settle
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        timeout: 1
+      changed_when: false
+
+    - name: Verify tunnel master running (ssh -O check)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
+      register: tun_check
+      changed_when: false
+
+    - name: Debug tunnel check
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tunnel_check.rc={{ tun_check.rc }}"
+          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
+
+    - name: Sanity confirm tunnel TCP reachability to DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug reachability result
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "nc.rc={{ nc_probe.rc }}"
+          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
+          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
+
+    - name: Stop if tunnel TCP check failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    # ---------------- Step 4: Determine working password for DEV2 ----------------
+    - name: Try both passfiles for DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
+            echo "$f"; exit 0;
+          fi
+        done
+        echo NONE
+      register: dev2_passfile_try
+      changed_when: false
+
+    - name: Save selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
+      changed_when: false
+
+    - name: Debug selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "dev2_passfile_used={{ dev2_passfile_used }}"
+
+    - name: Stop if no valid passfile found
+      ansible.builtin.meta: end_host
+      when: dev2_passfile_used == "NONE"
+
+    # ---------------- Step 5: Read firmware version on DEV2 ----------------
+    - name: Read firmware version from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
+      register: dev2_fwver
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show firmware version readout
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    # ---------------- Retry metadata + success evaluation ----------------
+    - name: Normalize retry metadata for indoor checker
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+
+    - name: Evaluate indoor success (firmware read OK)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        indoor_success: "{{ (dev2_fwver.rc | default(1) == 0)
+                            and ((dev2_fwver.stdout | default('') | trim) | length > 0)
+                            and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
+
+    # ---------------- Step 6: Publish result journal (+ tags on success) ----------------
+    # SUCCESS PATH: add tag, remove scheduled tag, and journal with "success"
+    - name: Build success journal payload
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
+            tunnel_port={{ _local_port }}
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_success
+      changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
+
+    - name: Build payload to add indoor-update-success tag (DEV1)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "indoor-update-success"
+
+    - name: Publish indoor-update-success tag
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add_success
+      changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
+
+    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_remove_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_remove"
+          task_result: "indoor-restart-scheduled"
+
+    - name: Publish indoor-restart-scheduled tag removal
+      when: tag_remove_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_remove_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_remove_sched
+      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
+
+    # --- Normalize firmware string and set custom field on success ---
+    - name: Capture raw firmware banner from DEV2 (for normalization)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
+      when:
+        - indoor_success | bool
+        - (fw_banner_raw | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_norm: >-
+          {{
+            fw_banner_raw
+            if (fw_banner_raw | lower is search('-r[0-9]+$'))
+            else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
+          }}
+
+    - name: Debug normalized firmware (indoor_fwver)
+      when: fw_norm is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
+
+    - name: Publish custom_field_set indoor_fwver
+      when:
+        - indoor_success | bool
+        - fw_norm is defined
+        - (fw_norm | length) > 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': fw_norm
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_customfield_fw
+      changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
+
+    # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
+    - name: Build failure journal payload (indoor firmware read failed)
+      when: not (indoor_success | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish failure journal to control queue
+      when: journal_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_fail
+      changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
+
+    - name: Compute retry parameters (10 minutes)
+      when: not (indoor_success | bool)
+      ansible.builtin.set_fact:
+        next_attempt: "{{ (attempt | int) + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: "{{ 600000 }}"
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_gaveup
+      changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
+
+    # Only schedule next attempt if we still have budget left
+    - name: Build delayed payload for next indoor attempt (10 min)
+      when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id | default('') }}"
+          original_emitted_at: "{{ original_emitted_at | default('') }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id | default('') }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+  post_tasks:
+    - name: Cleanup tunnel and temp IP
+      block:
+        - ansible.builtin.debug:
+            msg: "Cleaning up tunnel + temp IP"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP from DEV1
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          failed_when: false
+          changed_when: false
+

files/ansible-playbooks/basicpass

@@ -0,0 +1 @@
+wavewave

files/ansible-playbooks/basicpass2

@@ -0,0 +1 @@
+admin

files/ansible-playbooks/checkversioncli.yml

@@ -0,0 +1,18 @@
+---
+- name: Simple script to show current version
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: grab current version
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        cat /etc/banner | grep 2
+      register: banner_out
+      changed_when: true
+
+    - name: show the current version
+      ansible.builtin.debug:
+        msg: "{{ banner_out.stdout | trim }}"
+

files/ansible-playbooks/deploy-scroll24.yml

@@ -0,0 +1,277 @@
+---
+- name: Deploy scroll24 script and cron configuration
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # Version tag for scroll24 deployment (operator-controlled)
+    scroll24_version: "v1.1"
+
+    # Controller-side source paths (same place as wifidebug.sh)
+    src_script: "files/scroll24.sh"
+    src_cron_snippet: "files/crond-root-scroll24"
+
+    # Remote destinations
+    remote_script: "/root/scroll24.sh"
+    remote_crontab: "/etc/crontabs/root"
+    tmp_cron_snippet: "/tmp/crond-root-scroll24.snippet"
+    tmp_cron_new: "/tmp/cron.root.new"
+    crontab_backup_dir: "/etc/crontabs"
+
+    # RabbitMQ (same contract/style as your other playbooks)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    ##########################################################################
+    # 0. Controller sanity check for tools
+    ##########################################################################
+    - name: Verify controller tools (sshpass, sha256sum/busybox)
+      delegate_to: localhost
+      shell: |
+        command -v sshpass >/dev/null 2>&1 \
+        && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1)
+      args: { executable: /bin/bash }
+      register: ctrl_tools
+      changed_when: false
+      failed_when: ctrl_tools.rc != 0
+
+    ##########################################################################
+    # 1. Upload /root/scroll24.sh if changed (sha256 verified)
+    ##########################################################################
+    - name: Compute local sha256 of scroll24.sh
+      delegate_to: localhost
+      command: sha256sum {{ src_script }}
+      register: sha_local
+      changed_when: false
+
+    - name: Extract local sha256 digest (regex)
+      delegate_to: localhost
+      set_fact:
+        local_hash: "{{ (sha_local.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}"
+
+    - name: Compute remote sha256 of /root/scroll24.sh
+      raw: "sha256sum {{ remote_script }} 2>/dev/null || busybox sha256sum {{ remote_script }} 2>/dev/null || true"
+      register: sha_remote
+      changed_when: false
+      failed_when: false
+
+    - name: Extract remote sha256 digest (regex)
+      set_fact:
+        remote_hash: "{{ (sha_remote.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}"
+
+    - name: Decide if scroll24.sh needs upload
+      set_fact:
+        script_changed: "{{ (remote_hash | length == 0) or (local_hash != remote_hash) }}"
+
+    - name: Upload scroll24.sh via scp (if needed)
+      when: script_changed | bool
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ src_script }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_script }}
+      register: scp_scroll24
+      changed_when: true
+
+    - name: Ensure /root/scroll24.sh permissions and ownership
+      raw: "chown root:root {{ remote_script }} && chmod 0755 {{ remote_script }}"
+      changed_when: script_changed | bool
+
+    - name: Recompute remote sha256 after upload
+      when: script_changed | bool
+      raw: "sha256sum {{ remote_script }} || busybox sha256sum {{ remote_script }}"
+      register: sha_remote_after
+      changed_when: false
+      failed_when: false
+
+    - name: Extract remote-after sha256 digest (regex)
+      when: script_changed | bool
+      set_fact:
+        remote_hash_after: "{{ (sha_remote_after.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}"
+
+    - name: Fail if scroll24.sh sha256 mismatch after upload
+      when: script_changed | bool and (remote_hash_after | default('')) != (local_hash | default(''))
+      fail:
+        msg: "sha256 mismatch between controller and remote scroll24.sh"
+
+    ##########################################################################
+    # 1b. Extract the period (minutes) from the first line of the snippet
+    ##########################################################################
+    - name: Extract period minutes from first snippet line (*/N …)
+      delegate_to: localhost
+      shell: |
+        awk 'NR==1{
+          f=$1;
+          if (f ~ /^\*\/[0-9]+$/) { gsub("^\\*/","",f); print f; exit }
+          else if (f ~ /^[0-9]+$/) { print f; exit }
+          else { print ""; exit }
+        }' {{ src_cron_snippet }}
+      args: { executable: /bin/bash }
+      register: cron_period_cmd
+      changed_when: false
+      failed_when: false
+
+    - name: Set cron_period fact
+      delegate_to: localhost
+      set_fact:
+        cron_period: "{{ (cron_period_cmd.stdout | trim) }}"
+
+    ##########################################################################
+    # 2. Upload cron snippet to device /tmp (operator-provided content)
+    ##########################################################################
+    - name: Push cron snippet to device tmp
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ src_cron_snippet }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ tmp_cron_snippet }}
+      register: scp_snippet
+      changed_when: true
+
+    ##########################################################################
+    # 3. Crontab subset logic (only scroll24.sh lines)
+    ##########################################################################
+    - name: Ensure /etc/crontabs/root exists (with perms)
+      raw: |
+        if [ ! -f {{ remote_crontab }} ]; then
+          touch {{ remote_crontab }};
+        fi
+        chown root:root {{ remote_crontab }};
+        chmod 0644 {{ remote_crontab }};
+      changed_when: false
+
+    - name: Extract current scroll24 lines from crontab
+      raw: "grep -F 'scroll24.sh' {{ remote_crontab }} || true"
+      register: cron_subset
+      changed_when: false
+      failed_when: false
+
+    - name: Normalize controller snippet (for compare only)
+      delegate_to: localhost
+      shell: |
+        grep -v '^[[:space:]]*$' {{ src_cron_snippet }} \
+        | sed 's/[[:space:]]\+/ /g' | sed 's/[[:space:]]*$//' | sort -u
+      register: norm_snippet
+      changed_when: false
+
+    - name: Normalize device scroll24 subset (for compare only)
+      delegate_to: localhost
+      shell: |
+        printf "%s\n" "{{ cron_subset.stdout | default('') }}" \
+        | grep -v '^[[:space:]]*$' \
+        | sed 's/[[:space:]]\+/ /g' | sed 's/[[:space:]]*$//' | sort -u
+      register: norm_remote_subset
+      changed_when: false
+
+    - name: Decide if crontab needs update (subset compare)
+      delegate_to: localhost
+      set_fact:
+        crontab_changed: "{{ (norm_snippet.stdout | trim) != (norm_remote_subset.stdout | trim) }}"
+
+    - name: Backup current crontab (timestamped)
+      when: crontab_changed | bool
+      raw: "cp -a {{ remote_crontab }} {{ crontab_backup_dir }}/root.bak.$(date +%Y%m%d%H%M%S)"
+      changed_when: true
+
+    - name: Replace scroll24 lines in crontab (preserve all others; tidy splice)
+      when: crontab_changed | bool
+      raw: "grep -v 'scroll24\\.sh' {{ remote_crontab }} > {{ tmp_cron_new }} && awk 'BEGIN{for(i=1;i<=NR;i++)a[i]=$0} {a[NR]=$0} END{e=NR; while(e>0 && a[e] ~ /^[[:space:]]*$/){e--}; for(i=1;i<=e;i++) print a[i]}' {{ tmp_cron_new }} > {{ tmp_cron_new }}.trim && mv {{ tmp_cron_new }}.trim {{ tmp_cron_new }} && cat {{ tmp_cron_snippet }} >> {{ tmp_cron_new }} && printf '\\n' >> {{ tmp_cron_new }} && mv {{ tmp_cron_new }} {{ remote_crontab }} && chown root:root {{ remote_crontab }} && chmod 0644 {{ remote_crontab }}"
+      changed_when: true
+
+    ##########################################################################
+    # 4. Restart crond if script or cron changed (with :51–:59 guard)
+    ##########################################################################
+    - name: Check if restart required
+      set_fact:
+        need_restart: "{{ (script_changed | bool) or (crontab_changed | bool) }}"
+
+    - name: Get current seconds
+      when: need_restart | bool
+      raw: "date +%S"
+      register: nowsec
+      changed_when: false
+
+    - name: Sleep 10s if seconds 51-59
+      when: need_restart | bool and (nowsec.stdout | int >= 51)
+      pause:
+        seconds: 10
+
+    - name: Restart crond via move/move
+      when: need_restart | bool
+      raw: "mv /tmp/launchd/services/crond /root/crond && sleep 1 && mv /root/crond /tmp/launchd/services/crond"
+      register: crond_restart
+      changed_when: true
+      failed_when: false
+
+    - name: Verify crond is running
+      when: need_restart | bool
+      raw: "pgrep -f '/usr/sbin/crond' || busybox pgrep crond || echo missing"
+      register: crond_pid
+      changed_when: false
+      failed_when: false
+
+    ##########################################################################
+    # 5. Final journals to RabbitMQ (controller)
+    ##########################################################################
+    - name: Publish final scroll24 deployment journal
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'scroll24: '
+              ~ (script_changed | ternary('script installed/updated; ', 'script up-to-date; '))
+              ~ (crontab_changed | ternary('cron updated; ', 'cron already matches; '))
+              ~ (need_restart | ternary('crond restarted; ', 'crond unchanged; '))
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Set NetBox custom field scroll24 -> "<version>_<period>"
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'scroll24',
+            'task_result': scroll24_version ~ '_' ~ (cron_period | default(''))
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+

files/ansible-playbooks/full-upgrade-check.yml

@@ -0,0 +1,20 @@
+- import_playbook: update-rebootin222.yml
+  vars:
+    rebootin: 0
+
+- name:
+  hosts: all                # will match your limited hosts
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Wait 5 minutes
+      ansible.builtin.wait_for:
+        timeout: "{{ pause_minutes | default(300) }}"
+      run_once: true
+      delegate_to: localhost
+      check_mode: no
+      tags: always
+
+- import_playbook: checkversioncli.yml
+#- import_playbook: setntptime.yml
+

files/ansible-playbooks/full-upgrade-wifi14.yml

@@ -0,0 +1,21 @@
+- import_playbook: update-rebootin.yml
+  vars:
+    rebootin: 0
+
+- name:
+  hosts: all                # will match your limited hosts
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Wait 5 minutes
+      ansible.builtin.wait_for:
+        timeout: "{{ pause_minutes | default(300) }}"
+      run_once: true
+      delegate_to: localhost
+      check_mode: no
+      tags: always
+
+- import_playbook: checkversioncli.yml
+- import_playbook: setntptime.yml
+- import_playbook: wifidebug14.yml
+- import_playbook: system-stop-system-start.yml

files/ansible-playbooks/multissidfix-changessid.yml

@@ -0,0 +1,826 @@
+---
+- name: "Multi-SSID fix (phase 1: persist + runtime apply)"
+  hosts: all
+  gather_facts: false
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (same contract as wifidebug16.yml)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    # Reboot parachute delay (seconds)
+    parachute_delay_s: 600
+
+    # Local working paths (controller)
+    local_cfg:  "/tmp/{{ inventory_hostname }}_config.json"
+    local_new:  "/tmp/{{ inventory_hostname }}_config.json.new"
+    local_step: "/tmp/{{ inventory_hostname }}_config.step"
+    local_diff: "/tmp/{{ inventory_hostname }}_config.diff"
+    local_dd:   "/tmp/{{ inventory_hostname }}_dd.json"
+
+    # External scripts/filters (next to wifidebug.sh in your repo)
+    dd_script_src: "files/wirelessduediligence.sh"
+    f_1vap0_src:   "files/filter_1vap_id0.jq"
+    f_1vap1_src:   "files/filter_1vap_id1.jq"
+    f_2_01_src:    "files/filter_2vaps_0_1.jq"
+    f_2_10_src:    "files/filter_2vaps_1_0.jq"
+
+    # Controller tmp destinations
+    dd_script: "/tmp/wirelessduediligence.sh"
+    f_1vap0:   "/tmp/filter_1vap_id0.jq"
+    f_1vap1:   "/tmp/filter_1vap_id1.jq"
+    f_2_01:    "/tmp/filter_2vaps_0_1.jq"
+    f_2_10:    "/tmp/filter_2vaps_1_0.jq"
+
+    # Remote (device)
+    remote_cfg:    "/tmp/config.json"
+    remote_new:    "/tmp/config.json.new"
+    remote_backup: "/tmp/config.json.multissidfix1.backup"
+
+    # Optional new SSID name (set via env NEW_SSID to trigger rename)
+    new_ssid_name: "{{ lookup('env','NEW_SSID') | default('ikeja R5 a day hotspot', true) }}"
+
+  tasks:
+
+    ##########################################################################
+    # a) Connectivity + basic sanity
+    ##########################################################################
+    - name: SSH reachability probe
+      raw: "echo ping"
+      register: ping_result
+      ignore_errors: true
+
+    - name: Stop if SSH unreachable (soft-fail)
+      when: ping_result is failed
+      block:
+        - name: Journal soft-fail (no SSH)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — SSH not reachable'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Ensure remote /tmp/config.json exists
+      raw: "test -s {{ remote_cfg }}"
+      register: remote_cfg_check
+      changed_when: false
+
+    - name: Stop if /tmp/config.json missing (soft-fail)
+      when: remote_cfg_check.rc != 0
+      block:
+        - name: Journal soft-fail (missing config.json)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — missing /tmp/config.json'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Check controller tools (jq/sshpass/diff/sha256sum)
+      delegate_to: localhost
+      shell: |
+        command -v jq >/dev/null 2>&1 \
+        && command -v sshpass >/dev/null 2>&1 \
+        && command -v diff >/dev/null 2>&1 \
+        && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1)
+      args: { executable: /bin/bash }
+      register: ctrl_tools
+      changed_when: false
+
+    - name: Stop if controller tools missing (soft-fail)
+      when: ctrl_tools.rc != 0
+      block:
+        - name: Journal soft-fail (missing tools)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — controller missing jq/sshpass/diff/sha256sum'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # b) Due diligence (external script, run on controller)
+    ##########################################################################
+    - name: Copy due diligence script to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ dd_script_src }}"
+        dest: "{{ dd_script }}"
+        mode: "0755"
+
+    - name: Pull /tmp/config.json from device to controller
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_cfg }}
+        {{ local_cfg }}
+
+    - name: Run due diligence on controller
+      delegate_to: localhost
+      shell: "sh {{ dd_script | quote }} {{ local_cfg | quote }} > {{ local_dd | quote }}"
+      args: { executable: /bin/sh }
+      changed_when: false
+
+    - name: Parse due diligence JSON
+      delegate_to: localhost
+      set_fact:
+        dd: "{{ lookup('file', local_dd) | from_json }}"
+
+    # Normalize types for robust `when:` checks
+    - name: Extract & normalize dd facts
+      delegate_to: localhost
+      set_fact:
+        w0_total_i: "{{ (dd.radios.wifi0.vaps_total | int) }}"
+        w1_total_i: "{{ (dd.radios.wifi1.vaps_total | int) }}"
+        w0_idx_i:   "{{ (dd.radios.wifi0.indices | map('int') | list) }}"
+        w1_idx_i:   "{{ (dd.radios.wifi1.indices | map('int') | list) }}"
+
+    ##########################################################################
+    # c) Soft-fail: layout policy checks (one journal)
+    ##########################################################################
+    - name: Init soft-fail reasons (policy)
+      delegate_to: localhost
+      set_fact:
+        softfail_reasons: []
+
+    - name: wifi0 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w0_total_i | int) == 1 and (0 not in w0_idx_i and 1 not in w0_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 ONE_VAP but index not 0/1 (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w0_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 MORE_THAN_TWO_VAPS (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w1_total_i | int) == 1 and (0 not in w1_idx_i and 1 not in w1_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 ONE_VAP but index not 0/1 (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w1_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 MORE_THAN_TWO_VAPS (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: Publish soft-fail journal & stop host (policy)
+      when: (softfail_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (policy)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (policy) — ' ~ (softfail_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # d) Filters to controller tmp + candidate generation (per radio)
+    ##########################################################################
+    - name: Copy jq filters to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        mode: "0644"
+      loop:
+        - { src: "{{ f_1vap0_src }}", dest: "{{ f_1vap0 }}" }
+        - { src: "{{ f_1vap1_src }}", dest: "{{ f_1vap1 }}" }
+        - { src: "{{ f_2_01_src }}",  dest: "{{ f_2_01 }}"  }
+        - { src: "{{ f_2_10_src }}",  dest: "{{ f_2_10 }}"  }
+
+    - name: Start from current config as working file
+      delegate_to: localhost
+      copy:
+        src: "{{ local_cfg }}"
+        dest: "{{ local_step }}"
+        mode: "0644"
+
+    # wifi0 choice
+    - name: wifi0 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (0 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (1 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i == [1,0])
+      changed_when: true
+
+    # wifi1 choice
+    - name: wifi1 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (0 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (1 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i == [1,0])
+      changed_when: true
+
+    # Optional SSID rename (only if new_ssid_name is defined)
+    - name: Set SSID for the single enabled AP VAP per radio (wifi0 & wifi1)
+      delegate_to: localhost
+      shell: |
+        jq --arg SSID "{{ new_ssid_name }}" '
+          .wireless.radios |=
+            with_entries(
+              .value.vaps =
+                ((.value.vaps // []) | map(
+                  if (.mode=="ap" and (.enabled==true)) then
+                    (.ssid = $SSID)
+                  else .
+                  end
+                ))
+            )
+        ' {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}
+      args: { executable: /bin/bash }
+      when: new_ssid_name is defined
+      changed_when: true
+
+    - name: Debug normalized types
+      delegate_to: localhost
+      debug:
+        msg:
+          - "w0_total_i(type)={{ w0_total_i | type_debug }}  value={{ w0_total_i }}"
+          - "w1_total_i(type)={{ w1_total_i | type_debug }}  value={{ w1_total_i }}"
+
+    - name: Move working file to final candidate
+      delegate_to: localhost
+      copy:
+        src: "{{ local_step }}"
+        dest: "{{ local_new }}"
+        mode: "0644"
+
+    ##########################################################################
+    # e) Candidate checks — accumulate reasons; soft-fail once if any
+    ##########################################################################
+    - name: Init soft-fail reasons (candidate)
+      delegate_to: localhost
+      set_fact:
+        cand_reasons: []
+
+    - name: Check JSON validity
+      delegate_to: localhost
+      shell: "jq -e '.' {{ local_new | quote }} >/dev/null"
+      args: { executable: /bin/bash }
+      register: json_valid
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if JSON invalid
+      delegate_to: localhost
+      when: json_valid.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'invalid JSON syntax in candidate' ] }}"
+
+    - name: Semantic — wifi0 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 semantic fails
+      delegate_to: localhost
+      when: sem_w0.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi1 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 semantic fails
+      delegate_to: localhost
+      when: sem_w1.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi0 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 lbd check fails
+      delegate_to: localhost
+      when: sem_w0_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Semantic — wifi1 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 lbd check fails
+      delegate_to: localhost
+      when: sem_w1_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Get original file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_cfg }}"
+      register: stat_old
+      changed_when: false
+
+    - name: Get new file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_new }}"
+      register: stat_new
+      changed_when: false
+
+    - name: Add reason if size delta > 2%
+      delegate_to: localhost
+      when: stat_old.stat.size | int == 0 or
+            ( ((stat_new.stat.size | int) - (stat_old.stat.size | int)) | abs ) > ( (stat_old.stat.size | int) * 0.02 )
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'candidate size delta >2% (old=' ~ (stat_old.stat.size | string) ~ ', new=' ~ (stat_new.stat.size | string) ~ ')' ] }}"
+
+    - name: Build unified diff (first 200 lines)
+      delegate_to: localhost
+      shell: "diff -u {{ local_cfg | quote }} {{ local_new | quote }} | sed -n '1,200p' > {{ local_diff | quote }} || true"
+      args: { executable: /bin/bash }
+      changed_when: false
+
+    - name: Show unified diff (first 200 lines)
+      delegate_to: localhost
+      debug:
+        msg: "{{ lookup('file', local_diff) | default('(no diff output)') }}"
+
+    - name: sha256 (controller) of candidate
+      delegate_to: localhost
+      command: sha256sum {{ local_new }}
+      register: sha_local
+      changed_when: false
+
+    - name: Copy candidate to device temp
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ local_new }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_new }}
+      register: scp_new
+      changed_when: true
+
+    - name: sha256 (remote) of candidate
+      raw: "sha256sum {{ remote_new }} || busybox sha256sum {{ remote_new }}"
+      register: sha_remote
+      changed_when: false
+
+    - name: Add reason if sha256 mismatch
+      delegate_to: localhost
+      when: (sha_local.stdout.split()[0]) != (sha_remote.stdout.split()[0])
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'sha256 mismatch controller vs remote' ] }}"
+
+    - name: Detect SSIDs with NBSP (informational)
+      delegate_to: localhost
+      shell: |
+        nbsp="$(printf '\302\240')"
+        jq -r '..|objects|.ssid? // empty' {{ local_new | quote }} | grep -F "$nbsp" || true
+      args: { executable: /bin/bash }
+      register: nbsp_lines
+      changed_when: false
+
+    - name: Save NBSP report lines
+      set_fact:
+        ssid_nbsp_lines: "{{ nbsp_lines.stdout_lines | default([]) }}"
+
+    - name: Publish soft-fail journal & stop host (candidate issues)
+      when: (cand_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (candidate)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (candidate) — ' ~ (cand_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # f) Backup, parachute, promote, persist, apply, outcome
+    ##########################################################################
+    - name: Backup current config on device
+      raw: "cp -a {{ remote_cfg }} {{ remote_backup }}"
+      changed_when: true
+
+    # --- PRE-APPLY HEADS-UP ---
+    - name: Journal — candidate validated, will promote/persist/apply shortly
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: candidate passed checks; proceeding to backup + parachute + promote + sysconf -w + system-stop/start. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Arm reboot parachute (BusyBox reboot -d {{ parachute_delay_s }})
+      raw: "sh -c 'trap \"\" HUP; reboot -d {{ parachute_delay_s }} </dev/null >/dev/null 2>&1 &'"
+      changed_when: true
+      ignore_errors: true
+
+    - name: Promote candidate to active config
+      raw: "mv {{ remote_new }} {{ remote_cfg }} && chown root:root {{ remote_cfg }} && chmod 0644 {{ remote_cfg }}"
+      changed_when: true
+
+    # IMPORTANT: persist BEFORE runtime restart
+    - name: Persist config to flash (sysconf -w)
+      raw: "sysconf -w"
+      register: sysconf_write
+      changed_when: true
+      failed_when: false
+
+    # --- PRE-RESTART COMMIT ---
+    - name: Pre-restart journal (promoted & persisted; about to restart from controller)
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: promoted candidate, persisted (sysconf -w rc=' ~ (sysconf_write.rc | default('n/a')) | string ~ '), '
+              ~ 'system-stop/start launching from controller with 10s cap; '
+              ~ 'parachute -d ' ~ parachute_delay_s|string ~ 's armed. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+              ~ ', nbsp_ssids=' ~ (ssid_nbsp_lines|length)|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Launch restart from controller with 10s cap (SSH command)
+      delegate_to: localhost
+      shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+      args:
+        executable: /bin/bash
+      register: restart_kick
+      changed_when: true
+      failed_when: false
+
+    - name: Note restart_kick result (debug)
+      delegate_to: localhost
+      debug:
+        msg:
+          - "restart_kick.rc={{ restart_kick.rc }}"
+          - "stdout(last 5 lines): {{ (restart_kick.stdout_lines | default([]))[-5:] | default([]) }}"
+          - "stderr(last 5 lines): {{ (restart_kick.stderr_lines | default([]))[-5:] | default([]) }}"
+
+    # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+    - name: Probe SSH with nc (24 tries, 5s each)
+      delegate_to: localhost
+      shell: "nc -z -w 3 {{ ansible_host }} 22"
+      register: nc_probe
+      retries: 24
+      delay: 5
+      until: nc_probe.rc == 0
+      changed_when: false
+      failed_when: false
+
+    - name: Set ssh_up fact
+      delegate_to: localhost
+      set_fact:
+        ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+    # --- SUCCESS PATH ---
+    - name: Post-return actions (only if SSH came back)
+      when: ssh_up | bool
+      block:
+
+        - name: Disarm delayed reboot if present
+          raw: "pgrep -x reboot && kill -9 $(pgrep -x reboot) || true"
+          changed_when: true
+          ignore_errors: true
+          register: disarm_reboot
+
+        # Re-run simple semantic checks on the active config
+        - name: Device semantic — wifi0 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi0 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: sha256 (remote) of current active config (post-return)
+          raw: "sha256sum {{ remote_cfg }} || busybox sha256sum {{ remote_cfg }}"
+          register: sha_remote_after
+          changed_when: false
+          failed_when: false
+
+        - name: Decide post-return checks summary (controller side)
+          delegate_to: localhost
+          set_fact:
+            multissid_post_summary:
+              reboot_disarmed: "{{ (disarm_reboot is defined) and (disarm_reboot.rc is defined) and (disarm_reboot.rc in [0]) }}"
+              persisted_ok: "{{ (sysconf_write.rc | default(0)) == 0 }}"
+              dev_sem_ok:
+                w0_one: "{{ (dev_sem_w0.rc | default(1)) == 0 }}"
+                w1_one: "{{ (dev_sem_w1.rc | default(1)) == 0 }}"
+                w0_lbd: "{{ (dev_sem_w0_lbd.rc | default(1)) == 0 }}"
+                w1_lbd: "{{ (dev_sem_w1_lbd.rc | default(1)) == 0 }}"
+              cfg_hash_match: "{{ (sha_remote_after.stdout.split()[0] | default('')) == (sha_remote.stdout.split()[0] | default('')) }}"
+              old_size: "{{ stat_old.stat.size | default('n/a') }}"
+              new_size: "{{ stat_new.stat.size | default('n/a') }}"
+              nbsp_count: "{{ (ssid_nbsp_lines | default([])) | length }}"
+
+        - name: Publish final success journal (device returned; persisted; checks pass)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result':
+                  (
+                    'multissidfix: success — device is back; '
+                    ~ (multissid_post_summary.reboot_disarmed | ternary('parachute disarmed; ', 'parachute was not armed; '))
+                    ~ (multissid_post_summary.persisted_ok | ternary('persisted (sysconf -w); ', 'persist failed; '))
+                    ~ 'semantics: '
+                    ~ 'w0_one=' ~ (multissid_post_summary.dev_sem_ok.w0_one | string) ~ ', '
+                    ~ 'w1_one=' ~ (multissid_post_summary.dev_sem_ok.w1_one | string) ~ ', '
+                    ~ 'w0_lbd=' ~ (multissid_post_summary.dev_sem_ok.w0_lbd | string) ~ ', '
+                    ~ 'w1_lbd=' ~ (multissid_post_summary.dev_sem_ok.w1_lbd | string) ~ '; '
+                    ~ 'cfg_match=' ~ (multissid_post_summary.cfg_hash_match | string) ~ '; '
+                    ~ 'size(old/new)=' ~ (multissid_post_summary.old_size | string) ~ '/' ~ (multissid_post_summary.new_size | string) ~ '; '
+                    ~ 'nbsp_ssids=' ~ (multissid_post_summary.nbsp_count | string)
+                  )
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+
+    - name: Set NetBox custom field multissidfix=v1.1
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'multissidfix',
+            'task_result': 'v1.1'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    # --- FAILURE PATH (device did not return) ---
+    - name: Journal — restart/SSH probe failed
+      when: not ssh_up
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: restart issued; SSH did not return after 24 x 5s checks — leaving parachute active.'
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Stop host after failed SSH probe
+      when: not ssh_up
+      meta: end_host
+

files/ansible-playbooks/multissidfix.yml

@@ -0,0 +1,802 @@
+---
+- name: "Multi-SSID fix (phase 1: persist + runtime apply)"
+  hosts: all
+  gather_facts: false
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (same contract as wifidebug16.yml)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    # Reboot parachute delay (seconds)
+    parachute_delay_s: 600
+
+    # Local working paths (controller)
+    local_cfg:  "/tmp/{{ inventory_hostname }}_config.json"
+    local_new:  "/tmp/{{ inventory_hostname }}_config.json.new"
+    local_step: "/tmp/{{ inventory_hostname }}_config.step"
+    local_diff: "/tmp/{{ inventory_hostname }}_config.diff"
+    local_dd:   "/tmp/{{ inventory_hostname }}_dd.json"
+
+    # External scripts/filters (next to wifidebug.sh in your repo)
+    dd_script_src: "files/wirelessduediligence.sh"
+    f_1vap0_src:   "files/filter_1vap_id0.jq"
+    f_1vap1_src:   "files/filter_1vap_id1.jq"
+    f_2_01_src:    "files/filter_2vaps_0_1.jq"
+    f_2_10_src:    "files/filter_2vaps_1_0.jq"
+
+    # Controller tmp destinations
+    dd_script: "/tmp/wirelessduediligence.sh"
+    f_1vap0:   "/tmp/filter_1vap_id0.jq"
+    f_1vap1:   "/tmp/filter_1vap_id1.jq"
+    f_2_01:    "/tmp/filter_2vaps_0_1.jq"
+    f_2_10:    "/tmp/filter_2vaps_1_0.jq"
+
+    # Remote (device)
+    remote_cfg:    "/tmp/config.json"
+    remote_new:    "/tmp/config.json.new"
+    remote_backup: "/tmp/config.json.multissidfix1.backup"
+
+  tasks:
+
+    ##########################################################################
+    # a) Connectivity + basic sanity
+    ##########################################################################
+    - name: SSH reachability probe
+      raw: "echo ping"
+      register: ping_result
+      ignore_errors: true
+
+    - name: Stop if SSH unreachable (soft-fail)
+      when: ping_result is failed
+      block:
+        - name: Journal soft-fail (no SSH)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — SSH not reachable'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Ensure remote /tmp/config.json exists
+      raw: "test -s {{ remote_cfg }}"
+      register: remote_cfg_check
+      changed_when: false
+
+    - name: Stop if /tmp/config.json missing (soft-fail)
+      when: remote_cfg_check.rc != 0
+      block:
+        - name: Journal soft-fail (missing config.json)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — missing /tmp/config.json'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Check controller tools (jq/sshpass/diff/sha256sum)
+      delegate_to: localhost
+      shell: |
+        command -v jq >/dev/null 2>&1 \
+        && command -v sshpass >/dev/null 2>&1 \
+        && command -v diff >/dev/null 2>&1 \
+        && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1)
+      args: { executable: /bin/bash }
+      register: ctrl_tools
+      changed_when: false
+
+    - name: Stop if controller tools missing (soft-fail)
+      when: ctrl_tools.rc != 0
+      block:
+        - name: Journal soft-fail (missing tools)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — controller missing jq/sshpass/diff/sha256sum'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # b) Due diligence (external script, run on controller)
+    ##########################################################################
+    - name: Copy due diligence script to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ dd_script_src }}"
+        dest: "{{ dd_script }}"
+        mode: "0755"
+
+    - name: Pull /tmp/config.json from device to controller
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_cfg }}
+        {{ local_cfg }}
+
+    - name: Run due diligence on controller
+      delegate_to: localhost
+      shell: "sh {{ dd_script | quote }} {{ local_cfg | quote }} > {{ local_dd | quote }}"
+      args: { executable: /bin/sh }
+      changed_when: false
+
+    - name: Parse due diligence JSON
+      delegate_to: localhost
+      set_fact:
+        dd: "{{ lookup('file', local_dd) | from_json }}"
+
+    # Normalize types for robust `when:` checks
+    - name: Extract & normalize dd facts
+      delegate_to: localhost
+      set_fact:
+        w0_total_i: "{{ (dd.radios.wifi0.vaps_total | int) }}"
+        w1_total_i: "{{ (dd.radios.wifi1.vaps_total | int) }}"
+        w0_idx_i:   "{{ (dd.radios.wifi0.indices | map('int') | list) }}"
+        w1_idx_i:   "{{ (dd.radios.wifi1.indices | map('int') | list) }}"
+
+    ##########################################################################
+    # c) Soft-fail: layout policy checks (one journal)
+    ##########################################################################
+    - name: Init soft-fail reasons (policy)
+      delegate_to: localhost
+      set_fact:
+        softfail_reasons: []
+
+    - name: wifi0 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w0_total_i | int) == 1 and (0 not in w0_idx_i and 1 not in w0_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 ONE_VAP but index not 0/1 (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w0_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 MORE_THAN_TWO_VAPS (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w1_total_i | int) == 1 and (0 not in w1_idx_i and 1 not in w1_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 ONE_VAP but index not 0/1 (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w1_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 MORE_THAN_TWO_VAPS (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: Publish soft-fail journal & stop host (policy)
+      when: (softfail_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (policy)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (policy) — ' ~ (softfail_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # d) Filters to controller tmp + candidate generation (per radio)
+    ##########################################################################
+    - name: Copy jq filters to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        mode: "0644"
+      loop:
+        - { src: "{{ f_1vap0_src }}", dest: "{{ f_1vap0 }}" }
+        - { src: "{{ f_1vap1_src }}", dest: "{{ f_1vap1 }}" }
+        - { src: "{{ f_2_01_src }}",  dest: "{{ f_2_01 }}"  }
+        - { src: "{{ f_2_10_src }}",  dest: "{{ f_2_10 }}"  }
+
+    - name: Start from current config as working file
+      delegate_to: localhost
+      copy:
+        src: "{{ local_cfg }}"
+        dest: "{{ local_step }}"
+        mode: "0644"
+
+    # wifi0 choice
+    - name: wifi0 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (0 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (1 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i == [1,0])
+      changed_when: true
+
+    # wifi1 choice
+    - name: wifi1 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (0 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (1 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i == [1,0])
+      changed_when: true
+
+    - name: Debug normalized types
+      delegate_to: localhost
+      debug:
+        msg:
+          - "w0_total_i(type)={{ w0_total_i | type_debug }}  value={{ w0_total_i }}"
+          - "w1_total_i(type)={{ w1_total_i | type_debug }}  value={{ w1_total_i }}"
+
+    - name: Move working file to final candidate
+      delegate_to: localhost
+      copy:
+        src: "{{ local_step }}"
+        dest: "{{ local_new }}"
+        mode: "0644"
+
+    ##########################################################################
+    # e) Candidate checks — accumulate reasons; soft-fail once if any
+    ##########################################################################
+    - name: Init soft-fail reasons (candidate)
+      delegate_to: localhost
+      set_fact:
+        cand_reasons: []
+
+    - name: Check JSON validity
+      delegate_to: localhost
+      shell: "jq -e '.' {{ local_new | quote }} >/dev/null"
+      args: { executable: /bin/bash }
+      register: json_valid
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if JSON invalid
+      delegate_to: localhost
+      when: json_valid.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'invalid JSON syntax in candidate' ] }}"
+
+    - name: Semantic — wifi0 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 semantic fails
+      delegate_to: localhost
+      when: sem_w0.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi1 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 semantic fails
+      delegate_to: localhost
+      when: sem_w1.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi0 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 lbd check fails
+      delegate_to: localhost
+      when: sem_w0_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Semantic — wifi1 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 lbd check fails
+      delegate_to: localhost
+      when: sem_w1_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Get original file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_cfg }}"
+      register: stat_old
+      changed_when: false
+
+    - name: Get new file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_new }}"
+      register: stat_new
+      changed_when: false
+
+    - name: Add reason if size delta > 2%
+      delegate_to: localhost
+      when: stat_old.stat.size | int == 0 or
+            ( ((stat_new.stat.size | int) - (stat_old.stat.size | int)) | abs ) > ( (stat_old.stat.size | int) * 0.02 )
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'candidate size delta >2% (old=' ~ (stat_old.stat.size | string) ~ ', new=' ~ (stat_new.stat.size | string) ~ ')' ] }}"
+
+    - name: Build unified diff (first 200 lines)
+      delegate_to: localhost
+      shell: "diff -u {{ local_cfg | quote }} {{ local_new | quote }} | sed -n '1,200p' > {{ local_diff | quote }} || true"
+      args: { executable: /bin/bash }
+      changed_when: false
+
+    - name: Show unified diff (first 200 lines)
+      delegate_to: localhost
+      debug:
+        msg: "{{ lookup('file', local_diff) | default('(no diff output)') }}"
+
+    - name: sha256 (controller) of candidate
+      delegate_to: localhost
+      command: sha256sum {{ local_new }}
+      register: sha_local
+      changed_when: false
+
+    - name: Copy candidate to device temp
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ local_new }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_new }}
+      register: scp_new
+      changed_when: true
+
+    - name: sha256 (remote) of candidate
+      raw: "sha256sum {{ remote_new }} || busybox sha256sum {{ remote_new }}"
+      register: sha_remote
+      changed_when: false
+
+    - name: Add reason if sha256 mismatch
+      delegate_to: localhost
+      when: (sha_local.stdout.split()[0]) != (sha_remote.stdout.split()[0])
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'sha256 mismatch controller vs remote' ] }}"
+
+    - name: Detect SSIDs with NBSP (informational)
+      delegate_to: localhost
+      shell: |
+        nbsp="$(printf '\302\240')"
+        jq -r '..|objects|.ssid? // empty' {{ local_new | quote }} | grep -F "$nbsp" || true
+      args: { executable: /bin/bash }
+      register: nbsp_lines
+      changed_when: false
+
+    - name: Save NBSP report lines
+      set_fact:
+        ssid_nbsp_lines: "{{ nbsp_lines.stdout_lines | default([]) }}"
+
+    - name: Publish soft-fail journal & stop host (candidate issues)
+      when: (cand_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (candidate)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (candidate) — ' ~ (cand_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # f) Backup, parachute, promote, persist, apply, outcome
+    ##########################################################################
+    - name: Backup current config on device
+      raw: "cp -a {{ remote_cfg }} {{ remote_backup }}"
+      changed_when: true
+
+    # --- PRE-APPLY HEADS-UP ---
+    - name: Journal — candidate validated, will promote/persist/apply shortly
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: candidate passed checks; proceeding to backup + parachute + promote + sysconf -w + system-stop/start. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Arm reboot parachute (BusyBox reboot -d {{ parachute_delay_s }})
+      raw: "sh -c 'trap \"\" HUP; reboot -d {{ parachute_delay_s }} </dev/null >/dev/null 2>&1 &'"
+      changed_when: true
+      ignore_errors: true
+
+    - name: Promote candidate to active config
+      raw: "mv {{ remote_new }} {{ remote_cfg }} && chown root:root {{ remote_cfg }} && chmod 0644 {{ remote_cfg }}"
+      changed_when: true
+
+    # IMPORTANT: persist BEFORE runtime restart
+    - name: Persist config to flash (sysconf -w)
+      raw: "sysconf -w"
+      register: sysconf_write
+      changed_when: true
+      failed_when: false
+
+    # --- PRE-RESTART COMMIT ---
+    - name: Pre-restart journal (promoted & persisted; about to restart from controller)
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: promoted candidate, persisted (sysconf -w rc=' ~ (sysconf_write.rc | default('n/a')) | string ~ '), '
+              ~ 'system-stop/start launching from controller with 10s cap; '
+              ~ 'parachute -d ' ~ parachute_delay_s|string ~ 's armed. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+              ~ ', nbsp_ssids=' ~ (ssid_nbsp_lines|length)|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Launch restart from controller with 10s cap (SSH command)
+      delegate_to: localhost
+      shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+      args:
+        executable: /bin/bash
+      register: restart_kick
+      changed_when: true
+      failed_when: false
+
+    - name: Note restart_kick result (debug)
+      delegate_to: localhost
+      debug:
+        msg:
+          - "restart_kick.rc={{ restart_kick.rc }}"
+          - "stdout(last 5 lines): {{ (restart_kick.stdout_lines | default([]))[-5:] | default([]) }}"
+          - "stderr(last 5 lines): {{ (restart_kick.stderr_lines | default([]))[-5:] | default([]) }}"
+
+    # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+    - name: Probe SSH with nc (24 tries, 5s each)
+      delegate_to: localhost
+      shell: "nc -z -w 3 {{ ansible_host }} 22"
+      register: nc_probe
+      retries: 24
+      delay: 5
+      until: nc_probe.rc == 0
+      changed_when: false
+      failed_when: false
+
+    - name: Set ssh_up fact
+      delegate_to: localhost
+      set_fact:
+        ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+    # --- SUCCESS PATH ---
+    - name: Post-return actions (only if SSH came back)
+      when: ssh_up | bool
+      block:
+
+        - name: Disarm delayed reboot if present
+          raw: "pgrep -x reboot && kill -9 $(pgrep -x reboot) || true"
+          changed_when: true
+          ignore_errors: true
+          register: disarm_reboot
+
+        # Re-run simple semantic checks on the active config
+        - name: Device semantic — wifi0 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi0 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: sha256 (remote) of current active config (post-return)
+          raw: "sha256sum {{ remote_cfg }} || busybox sha256sum {{ remote_cfg }}"
+          register: sha_remote_after
+          changed_when: false
+          failed_when: false
+
+        - name: Decide post-return checks summary (controller side)
+          delegate_to: localhost
+          set_fact:
+            multissid_post_summary:
+              reboot_disarmed: "{{ (disarm_reboot is defined) and (disarm_reboot.rc is defined) and (disarm_reboot.rc in [0]) }}"
+              persisted_ok: "{{ (sysconf_write.rc | default(0)) == 0 }}"
+              dev_sem_ok:
+                w0_one: "{{ (dev_sem_w0.rc | default(1)) == 0 }}"
+                w1_one: "{{ (dev_sem_w1.rc | default(1)) == 0 }}"
+                w0_lbd: "{{ (dev_sem_w0_lbd.rc | default(1)) == 0 }}"
+                w1_lbd: "{{ (dev_sem_w1_lbd.rc | default(1)) == 0 }}"
+              cfg_hash_match: "{{ (sha_remote_after.stdout.split()[0] | default('')) == (sha_remote.stdout.split()[0] | default('')) }}"
+              old_size: "{{ stat_old.stat.size | default('n/a') }}"
+              new_size: "{{ stat_new.stat.size | default('n/a') }}"
+              nbsp_count: "{{ (ssid_nbsp_lines | default([])) | length }}"
+
+        - name: Publish final success journal (device returned; persisted; checks pass)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result':
+                  (
+                    'multissidfix: success — device is back; '
+                    ~ (multissid_post_summary.reboot_disarmed | ternary('parachute disarmed; ', 'parachute was not armed; '))
+                    ~ (multissid_post_summary.persisted_ok | ternary('persisted (sysconf -w); ', 'persist failed; '))
+                    ~ 'semantics: '
+                    ~ 'w0_one=' ~ (multissid_post_summary.dev_sem_ok.w0_one | string) ~ ', '
+                    ~ 'w1_one=' ~ (multissid_post_summary.dev_sem_ok.w1_one | string) ~ ', '
+                    ~ 'w0_lbd=' ~ (multissid_post_summary.dev_sem_ok.w0_lbd | string) ~ ', '
+                    ~ 'w1_lbd=' ~ (multissid_post_summary.dev_sem_ok.w1_lbd | string) ~ '; '
+                    ~ 'cfg_match=' ~ (multissid_post_summary.cfg_hash_match | string) ~ '; '
+                    ~ 'size(old/new)=' ~ (multissid_post_summary.old_size | string) ~ '/' ~ (multissid_post_summary.new_size | string) ~ '; '
+                    ~ 'nbsp_ssids=' ~ (multissid_post_summary.nbsp_count | string)
+                  )
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+
+    - name: Set NetBox custom field multissidfix=v1
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'multissidfix',
+            'task_result': 'v1'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    # --- FAILURE PATH (device did not return) ---
+    - name: Journal — restart/SSH probe failed
+      when: not ssh_up
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: restart issued; SSH did not return after 24 x 5s checks — leaving parachute active.'
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Stop host after failed SSH probe
+      when: not ssh_up
+      meta: end_host

files/ansible-playbooks/persuasive-upgrade.yml

@@ -0,0 +1,330 @@
+---
+- name: Persuasive / hunting upgrade orchestrator
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ config
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+
+    # Exchanges / queues
+    work_exchange: "deviceconfig"
+    work_routing_key: "deviceconfig"           # immediate work path
+    holding_exchange: "deviceconfig.holding"
+    holding_routing_key: "persuasive"          # reschedules go to persuasive holding
+    control_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_timeout: 10
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+
+    # Policy defaults (overridable via -e from task_options)
+    pu_period_default: "30m"
+    pu_attempts_default: 48
+    pu_untilhours_default: "72h"
+
+  tasks:
+    - name: Normalize inputs (no self-referential defaults)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        pu_period: "{{ pu_period | default(pu_period_default) }}"
+        pu_attempts: "{{ (pu_attempts | default(pu_attempts_default)) | int }}"
+        pu_untilhours: "{{ pu_untilhours | default(pu_untilhours_default) }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default(lookup('pipe','date -u +%FT%TZ')) }}"
+        controller_now_iso: "{{ lookup('pipe','date -u +%FT%TZ') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "pu_period={{ pu_period }}"
+          - "pu_attempts={{ pu_attempts }}"
+          - "pu_untilhours={{ pu_untilhours }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "controller_now={{ controller_now_iso }}"
+
+    # Optional time budget
+    - name: Compute budget_ms (supports d/h/m/s in pu_untilhours)
+      ansible.builtin.set_fact:
+        budget_ms: >-
+          {{
+            (
+              (
+                (pu_untilhours | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
+                (pu_untilhours | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
+                (pu_untilhours | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
+                (pu_untilhours | regex_findall('([0-9]+)s') | first | default('0')) | int
+              ) * 1000
+            )
+          }}
+
+    - name: Compute elapsed since original emission (ms)
+      ansible.builtin.set_fact:
+        elapsed_ms: >-
+          {{
+            (
+              (lookup('pipe', 'date -u -d ' ~ controller_now_iso ~ ' +%s') | int) -
+              (lookup('pipe', 'date -u -d ' ~ original_emitted_at ~ ' +%s') | int)
+            ) * 1000
+          }}
+
+    # Helpers for final summary line
+    - name: Compute period_sec from pu_period (supports d/h/m/s)
+      ansible.builtin.set_fact:
+        period_sec: >-
+          {{
+            (
+              (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
+              (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
+              (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
+              (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
+            )
+          }}
+
+    - name: Compute budget_hours string (one decimal)
+      ansible.builtin.set_fact:
+        budget_hours_str: "{{ '%.1f' | format( (budget_ms | int) / 3600000.0 ) }}"
+
+    - name: Gave up due to time budget
+      when: (elapsed_ms | int) >= (budget_ms | int)
+      delegate_to: localhost
+      block:
+        - name: Journal final give-up (unified message)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: >-
+                {{
+                  {
+                    "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                    "task_name": "journal_add",
+                    "task_result": (
+                      "device wasn't reachable for " ~ (attempt | string) ~
+                      " attempts, each " ~ (period_sec | string) ~
+                      " seconds for " ~ budget_hours_str ~
+                      " hours. backing off, won't persuade it more. Schedule again if needed (limit: time budget exhausted)"
+                    )
+                  } | to_json
+                }}
+              payload_encoding: "string"
+          register: rmq_j_budget
+          changed_when: (rmq_j_budget.json is defined) and (rmq_j_budget.json.routed | default(false) | bool)
+
+        - ansible.builtin.meta: end_host
+
+    # Attempt cap
+    - name: Gave up due to attempts cap
+      when: (attempt | int) >= (pu_attempts | int)
+      delegate_to: localhost
+      block:
+        - name: Journal final give-up (unified message)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: >-
+                {{
+                  {
+                    "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                    "task_name": "journal_add",
+                    "task_result": (
+                      "device wasn't reachable for " ~ (attempt | string) ~
+                      " attempts, each " ~ (period_sec | string) ~
+                      " seconds for " ~ budget_hours_str ~
+                      " hours. backing off, won't persuade it more. Schedule again if needed (limit: attempts cap reached)"
+                    )
+                  } | to_json
+                }}
+              payload_encoding: "string"
+          register: rmq_j_gaveup
+          changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+        - ansible.builtin.meta: end_host
+
+    # Probe
+    - name: Check TCP/{{ tcp_port }} via nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # ONLINE → hand off to normal path
+    - name: Publish upgrade-confirmed to deviceconfig (immediate)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ work_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ work_routing_key }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "upgrade-confirmed"
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_start
+      changed_when: (rmq_start.json is defined) and (rmq_start.json.routed | default(false) | bool)
+
+    - name: "Journal: online, starting upgrade"
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": ("persuasive-upgrade: device online, starting upgrade-confirmed now. Original=" ~ original_emitted_at)
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_j_start
+      changed_when: (rmq_j_start.json is defined) and (rmq_j_start.json.routed | default(false) | bool)
+
+    - ansible.builtin.meta: end_host
+      when: nc_probe.rc == 0
+
+    # OFFLINE → reschedule into persuasive holding
+    - name: Compute TTL ms from pu_period (supports d/h/m/s)
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        ttl_ms: >-
+          {{
+            (
+              (
+                (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
+                (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
+                (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
+                (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
+              ) * 1000
+            )
+          }}
+
+    - name: Compute next_attempt
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+
+    - name: Build next task_options (carry policy + increment attempt)
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        next_task_options: >-
+          -e pu_period={{ pu_period }}
+          -e pu_attempts={{ pu_attempts }}
+          -e pu_untilhours={{ pu_untilhours }}
+          -e attempt={{ next_attempt }}
+          -e original_emitted_at='{{ original_emitted_at }}'
+
+    - name: Decide if we should emit the reschedule journal this attempt
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        _pu_journal_this_try: "{{ (attempt | int) in [1, 2] or ((attempt | int) % 10 == 0) }}"
+
+    - name: "Publish delayed persuasive-upgrade to holding (routing: persuasive)"
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers:
+              x-delay: "{{ ttl_ms | int }}"
+          routing_key: "{{ holding_routing_key }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "persuasive-upgrade",
+                "task_options": (next_task_options | trim)
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: "Journal: offline, rescheduled"
+      when: nc_probe.rc != 0 and (_pu_journal_this_try | bool)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": ("persuasive-upgrade: device offline, rescheduling (attempt " ~ attempt ~ "/" ~ pu_attempts ~ ", next in " ~ pu_period ~ "). Original=" ~ original_emitted_at)
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_j_off
+      changed_when: (rmq_j_off.json is defined) and (rmq_j_off.json.routed | default(false) | bool)
+
+    - ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0

files/ansible-playbooks/remove-scroll24.yml

@@ -0,0 +1,158 @@
+---
+- name: Remove scroll24 script and cron references
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    remote_script: "/root/scroll24.sh"
+    remote_crontab: "/etc/crontabs/root"
+    tmp_cron_new: "/tmp/cron.root.new"
+    crontab_backup_dir: "/etc/crontabs"
+
+    # RabbitMQ (same contract you use elsewhere)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    ##########################################################################
+    # 1) Make sure crontab file exists (don’t create noise)
+    ##########################################################################
+    - name: Ensure crontab file exists (with perms)
+      raw: |
+        if [ ! -f {{ remote_crontab }} ]; then
+          touch {{ remote_crontab }};
+        fi
+        chown root:root {{ remote_crontab }};
+        chmod 0644 {{ remote_crontab }};
+      changed_when: false
+
+    ##########################################################################
+    # 2) Check whether any scroll24 entries exist
+    ##########################################################################
+    - name: Detect existing scroll24 lines
+      raw: "grep -F 'scroll24.sh' {{ remote_crontab }} || true"
+      register: cron_subset
+      changed_when: false
+      failed_when: false
+
+    - name: Decide if crontab needs cleanup
+      set_fact:
+        crontab_changed: "{{ (cron_subset.stdout | trim) != '' }}"
+
+    - name: Backup current crontab (timestamped)
+      when: crontab_changed | bool
+      raw: "cp -a {{ remote_crontab }} {{ crontab_backup_dir }}/root.bak.$(date +%Y%m%d%H%M%S)"
+      changed_when: true
+
+    - name: Remove scroll24 lines from crontab (preserve others; tidy EOF)
+      when: crontab_changed | bool
+      raw: "grep -v 'scroll24\\.sh' {{ remote_crontab }} > {{ tmp_cron_new }} && awk 'BEGIN{for(i=1;i<=NR;i++)a[i]=$0} {a[NR]=$0} END{e=NR; while(e>0 && a[e] ~ /^[[:space:]]*$/){e--}; for(i=1;i<=e;i++) print a[i]}' {{ tmp_cron_new }} > {{ tmp_cron_new }}.trim && mv {{ tmp_cron_new }}.trim {{ tmp_cron_new }} && printf '\\n' >> {{ tmp_cron_new }} && mv {{ tmp_cron_new }} {{ remote_crontab }} && chown root:root {{ remote_crontab }} && chmod 0644 {{ remote_crontab }}"
+      changed_when: true
+
+    ##########################################################################
+    # 3) Remove /root/scroll24.sh if present
+    ##########################################################################
+    - name: Check if /root/scroll24.sh exists
+      raw: "[ -f {{ remote_script }} ] && echo PRESENT || echo ABSENT"
+      register: script_check
+      changed_when: false
+
+    - name: Remove /root/scroll24.sh
+      when: (script_check.stdout | trim) == 'PRESENT'
+      raw: "rm -f {{ remote_script }}"
+      register: rm_script
+      changed_when: true
+
+    - name: Flag script_removed
+      set_fact:
+        script_removed: "{{ ((script_check.stdout | trim) == 'PRESENT') }}"
+
+    ##########################################################################
+    # 4) If crontab changed, restart crond (with :51–:59 guard)
+    ##########################################################################
+    - name: Get current seconds
+      when: crontab_changed | bool
+      raw: "date +%S"
+      register: nowsec
+      changed_when: false
+
+    - name: Sleep 10s if seconds 51-59
+      when: crontab_changed | bool and (nowsec.stdout | int >= 51)
+      pause:
+        seconds: 10
+
+    - name: Restart crond via move/move
+      when: crontab_changed | bool
+      raw: "mv /tmp/launchd/services/crond /root/crond && sleep 1 && mv /root/crond /tmp/launchd/services/crond"
+      register: crond_restart
+      changed_when: true
+      failed_when: false
+
+    - name: Verify crond is running
+      when: crontab_changed | bool
+      raw: "pgrep -f '/usr/sbin/crond' || busybox pgrep crond || echo missing"
+      register: crond_pid
+      changed_when: false
+      failed_when: false
+
+    ##########################################################################
+    # 5) Journal + reset custom field
+    ##########################################################################
+    - name: Publish removal journal to control queue
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result':
+              (
+                'scroll24: removal — '
+                ~ (crontab_changed | ternary('crontab cleaned; ', 'no crontab refs; '))
+                ~ (script_removed   | ternary('script deleted; ', 'script not present; '))
+                ~ (crontab_changed | ternary('crond restarted', 'crond unchanged'))
+              )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Clear NetBox custom field scroll24
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'scroll24',
+            'task_result': 'nomore'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/resetradios.yml

@@ -0,0 +1,33 @@
+---
+- name: Reset radios on a host and show brief status
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Bounce ath0 and ath1 with ifconfig
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        ifconfig ath0 down
+        sleep 2
+        ifconfig ath0 up
+        ifconfig ath1 down
+        sleep 1
+        ifconfig ath1 up
+      register: reset_out
+      changed_when: true
+
+    - name: Wait 2s for interfaces to settle (controller-side)
+      ansible.builtin.pause:
+        seconds: 2
+
+    - name: Grab brief ath0/ath1 status (header + one following line)
+      ansible.builtin.raw: |
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        iwconfig 2>/dev/null | grep -E '^(ath0|ath1)\b' -A 1
+      register: iw_out
+      changed_when: false
+
+    - name: Show ath0/ath1 status
+      ansible.builtin.debug:
+        msg: "{{ iw_out.stdout | trim }}"

files/ansible-playbooks/setntptime.yml

@@ -0,0 +1,39 @@
+---
+- name: Simple script to show current version
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: get the date
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        date
+      register: date_out
+      changed_when: true
+
+
+    - name: show the current date
+      ansible.builtin.debug:
+        msg: "{{ date_out.stdout | trim }}"
+
+    - name: use sntp -S to set system time
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        sntp -S time.ikeja.co.za
+      changed_when: true
+
+    - name: get the date
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        date
+      register: date_out
+      changed_when: true
+
+
+    - name: show the current date
+      ansible.builtin.debug:
+        msg: "{{ date_out.stdout | trim }}"
+

files/ansible-playbooks/system-check-version.yml

@@ -0,0 +1,152 @@
+---
+- name: System firmware version check (banner probe + journal)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ (match your existing defaults)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults (kept identical to your afterupgrade_check.yml)
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+  tasks:
+    # -------- Fast TCP reachability probe (controller-side), unchanged style --------
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # If TCP failed → emit a single journal line and stop
+    - name: Build TCP-fail journal payload
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tcp_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            system check: TCP {{ tcp_port }} unreachable (nc failed)
+
+    - name: Publish TCP-fail journal
+      when: tcp_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tcp_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tcp_fail_resp
+      changed_when: (rmq_tcp_fail_resp.json is defined) and (rmq_tcp_fail_resp.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP failure
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    # -------- SSH banner probe (controller-side), EXACT command reused --------
+    - name: Probe banner via SSH from controller (classic extraction)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    # SSH error → journal and stop
+    - name: Build SSH-fail journal payload
+      when: banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        ssh_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            system check: SSH error - {{ (banner_probe.stderr | default('') | trim) }}
+
+    - name: Publish SSH-fail journal
+      when: ssh_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssh_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssh_fail_resp
+      changed_when: (rmq_ssh_fail_resp.json is defined) and (rmq_ssh_fail_resp.json.routed | default(false) | bool)
+
+    - name: Stop host after SSH failure
+      when: banner_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    # -------- Success: banner line captured → journal with "system check:" prefix --------
+    - name: Build success journal payload (banner captured)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        sc_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            system check: Banner='{{ (banner_probe.stdout | default('') | trim) }}'
+
+    - name: Publish success journal
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ sc_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_sc_success_resp
+      changed_when: (rmq_sc_success_resp.json is defined) and (rmq_sc_success_resp.json.routed | default(false) | bool)
+

files/ansible-playbooks/system-stop-system-start.yml

@@ -0,0 +1,19 @@
+---
+- name: Simple script restart all processes
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: execute and grab output
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        system-stop ; sleep 1; system-start
+      register: ssss_out
+      changed_when: true
+
+    - name: show the output
+      ansible.builtin.debug:
+        msg: "{{ ssss_out.stdout | trim }}"
+
+

files/ansible-playbooks/update-indoor.yml-bckp1

@@ -0,0 +1,997 @@
+---
+# update-secondline.yml
+- name: Second-line upgrade tunneled noninvasive engine
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Busybox-safe PATH prefix for all remote raw calls on DEV1
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # DEV1 credentials (stable, like rebootin222)
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    # Tunnel target DEV2 behind DEV1
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    # Temp IP we add to DEV1 so it can reach DEV2
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+
+    # DEV2 behind the tunnel
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # SSH options used from controller
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
+
+    # Image to stage on DEV2 noninvasive we do not run update -w here
+    image_filename: "fox200-2.2.1-r6801.bin"
+    image_md5: "56b7211709de617e058b98d4204e2562"
+    # Optional SHA256; leave empty to skip SHA256 checks
+    image_sha256: ""
+    dev2_image_dir: "/tmp"
+    dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
+
+    # Reboot delay in minutes integer string if set we will schedule reboot on DEV2
+    rebootin: ""
+
+    # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  pre_tasks:
+    # ------------------------------- Hostname sanity DEV1 -------------------------------
+    - name: Read DEV1 hostname from env busybox safe
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 connected hostname differs from inventory
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 idempotent treat File exists as OK
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug add ip results
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ----------------------------
+    - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+
+    - name: Set dev2_mac from fdb scan (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Remove current ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP for DEV2 on DEV1
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug static ARP result
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Skip ARP add because MAC not found
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- Local tunnel preparation ----------------------------
+    - name: Pick a free local port for the tunnel controller
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+            for i in $(seq 1 25); do
+                p="$(shuf -i 20000-39999 -n 1)"
+                if command -v ss >/dev/null 2>&1; then
+                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                        echo "$p"; return 0
+                    fi
+                else
+                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                        echo "$p"; return 0
+                    fi
+                fi
+            done
+            return 1
+        }
+        pick
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control directory
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Debug chosen tunnel params
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "local_port={{ _local_port }}"
+          - "ctrl_dir={{ _ctrl_dir }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+
+    - name: issue arping with own IP
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
+    # ---------------------------- Start SSH local forward via DEV1 ----------------------------
+    - name: Start SSH tunnel via DEV1 background ControlMaster
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+            -M -S "{{ _ctrl_sock }}" \
+            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+            "${USER}@${HOST}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    - name: Small delay for tunnel to settle
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        timeout: 1
+      changed_when: false
+
+    - name: Verify tunnel master running proper O check with destination
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
+      register: tun_check
+      changed_when: false
+
+    - name: Debug tunnel check
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tunnel_check.rc={{ tun_check.rc }}"
+          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
+
+    # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
+    - name: Sanity confirm tunnel TCP reachability to DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug reachability result
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "nc.rc={{ nc_probe.rc }}"
+          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
+          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
+
+    - name: Stop if tunnel TCP check failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    - name: Sanity show presence and permissions of passfiles on controller
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
+      register: dev2_ls
+      changed_when: false
+
+    - name: Debug passfiles listing
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ (dev2_ls.stdout | default('')) | trim }}"
+          - "{{ (dev2_ls.stderr | default('')) | trim }}"
+
+    # ---------------------------- Pick DEV2 password root only ----------------------------
+    - name: Try DEV2 login with basicpass root
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass as working passfile
+      when: dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login with basicpass2 root only if first failed
+      when: dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 as working passfile
+      when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Set dev2_passfile_used to NONE if neither worked
+      when: dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    - name: Debug selected DEV2 passfile or NONE
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_passfile_used={{ dev2_passfile_used }}"
+          - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}"
+          - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}"
+
+    - name: Read DEV2 hostname via tunnel (busybox-safe)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
+      args:
+        executable: /bin/bash
+      register: dev2_host_read
+      changed_when: false
+
+    - name: Normalize hostnames for strict compare
+      ansible.builtin.set_fact:
+        _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+
+    - name: Debug hostname bytes (hex)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "inv={{ _inv_hn | tojson }}"
+          - "dev1={{ _dev1_hn | tojson }}"
+          - "dev2={{ _dev2_hn | tojson }}"
+
+    # ---------------- Hostname equality guard (soft-journal and stop) ----------------
+    - name: Enforce DEV2 hostname equals inventory and DEV1
+      block:
+        - name: Fail if DEV2 hostname differs from inventory/DEV1
+          ansible.builtin.fail:
+            msg: >
+              Hostname mismatch: DEV2='{{ _dev2_hn }}',
+              inventory='{{ _inv_hn }}',
+              DEV1='{{ _dev1_hn }}'
+          when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
+      rescue:
+        - name: Initialize journal array for hostname mismatch
+          ansible.builtin.set_fact:
+            _journal: []
+            _blocked: true
+            _prep_blocked: false
+          delegate_to: localhost
+
+        - name: Append hostname mismatch to journal
+          ansible.builtin.set_fact:
+            _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}"
+          delegate_to: localhost
+
+        - name: Build control queue payload for indoor aborted journal (hostname)
+          ansible.builtin.set_fact:
+            journal_indoor_aborted:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
+          delegate_to: localhost
+
+        - name: Publish indoor aborted journal to control queue (hostname)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_indoor_aborted | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_indoor_aborted_hn_resp
+          changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_indoor_aborted_hn_resp.status != 200) or
+            (rmq_journal_indoor_aborted_hn_resp.json is not defined) or
+            (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop host after hostname mismatch
+          ansible.builtin.meta: end_host
+
+    # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
+    - name: Initialize journal and flags
+      ansible.builtin.set_fact:
+        _journal: []
+        _prep_blocked: false
+        _blocked: false
+
+    - name: Build specific marker filename (no .bin)
+      ansible.builtin.set_fact:
+        _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}"
+
+    - name: Check for any preparation markers on DEV2 (wildcard count)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "ls /tmp/prepared_for* 2>/dev/null | wc -l"
+      args:
+        executable: /bin/bash
+      register: dev2_prep_count
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if any prep markers already present
+      when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
+
+    # ---------------------------- DEV2 version firmux primary check ----------------------------
+    - name: Read DEV2 firmux release
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      args:
+        executable: /bin/bash
+      register: dev2_firmux
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug firmux content DEV2
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+
+    # ---------------------------- Normalize rebootin early ----------------------------
+    - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}"
+        _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}"
+        _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}"
+        _reboot_minutes: >-
+          {{
+            0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'])
+            else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1)
+          }}
+
+    - name: "Normalize rebootin (phase 2: requested flag)"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}"
+
+    # ---------------------------- Journal: indoor start (after we know we can proceed) ----------------------------
+    - name: Build control queue payload for indoor start journal
+      ansible.builtin.set_fact:
+        journal_indoor_start:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: Dev2 is reachable, starting with update.
+            image={{ image_filename }}, md5={{ image_md5 }},
+            firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }},
+            reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }}
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+
+    - name: Publish indoor start journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_start | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_start_resp
+      changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_start_resp.status != 200) or
+        (rmq_journal_indoor_start_resp.json is not defined) or
+        (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
+      when: journal_indoor_start is defined
+      delegate_to: localhost
+
+    # ---------------------------- Stage image on DEV2 no write ----------------------------
+    - name: Check local presence of image file
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ image_filename }}"
+      register: local_img
+
+    - name: Soft-block if image missing locally
+      when: not local_img.stat.exists
+      ansible.builtin.set_fact:
+        _blocked: true
+        _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
+
+    - name: Compute local md5 of the image
+      when: local_img.stat.exists
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        md5sum "{{ image_filename }}" | awk '{print $1}'
+      register: local_md5
+      changed_when: false
+
+    - name: Verify local md5 matches expected
+      when: local_img.stat.exists
+      delegate_to: localhost
+      ansible.builtin.assert:
+        that:
+          - (local_md5.stdout | trim) == image_md5
+        fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
+        success_msg: "Local md5 matches expected"
+
+    # Optional SHA256 local
+    - name: Compute local sha256 of the image (if provided)
+      when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sha256sum "{{ image_filename }}" | awk '{print $1}'
+      register: local_sha256
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if local sha256 mismatch or unavailable
+      when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
+
+    # fw_printenv health before upload (soft-fail)
+    - name: Read fw_printenv size (line count) on DEV2
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "(fw_printenv 2>/dev/null | wc -l) || echo 0"
+      args:
+        executable: /bin/bash
+      register: dev2_fwenv_wc
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if fw_printenv too small
+      when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
+
+    - name: Check existing image md5 on DEV2 NOFILE if absent
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
+      args:
+        executable: /bin/bash
+      register: dev2_md5_before
+      changed_when: false
+
+    - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" scp \
+          -P "$PORT" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/"
+      args:
+        executable: /bin/bash
+      register: scp_push
+      changed_when: true
+
+    - name: Compute md5 of image on DEV2 after copy or if exists
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
+      args:
+        executable: /bin/bash
+      register: dev2_md5_after
+      changed_when: false
+
+    - name: Verify DEV2 md5 matches expected (soft-fail journal)
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
+
+    # Optional SHA256 remote
+    - name: Compute sha256 of image on DEV2 (if provided)
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
+      args:
+        executable: /bin/bash
+      register: dev2_sha256_after
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if DEV2 sha256 mismatch/unavailable
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
+
+    - name: Run noninvasive update check on DEV2 update c
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "update -c '{{ dev2_image_path }}' 2>&1 || true"
+      args:
+        executable: /bin/bash
+      register: dev2_update_check
+      changed_when: false
+
+    - name: Debug update c output DEV2
+      when: dev2_update_check is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
+
+    - name: Soft-block if update -c did not say valid
+      when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
+
+    # ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
+    - name: Build control queue payload for indoor aborted journal
+      ansible.builtin.set_fact:
+        journal_indoor_aborted:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
+      when: (_blocked | default(false))
+      delegate_to: localhost
+
+    - name: Publish indoor aborted journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_aborted | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_aborted_resp
+      changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_aborted_resp.status != 200) or
+        (rmq_journal_indoor_aborted_resp.json is not defined) or
+        (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
+      when: journal_indoor_aborted is defined
+      delegate_to: localhost
+
+    # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
+    - name: Upgrade write and flip block
+      when: not (_blocked | default(false))
+      block:
+        - name: Write image on DEV2 (this will take a while)
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=0 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "update -w '{{ dev2_image_path }}'"
+          args:
+            executable: /bin/bash
+          register: dev2_up_write
+          changed_when: true
+          failed_when: dev2_up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition on DEV2
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "fw_printenv active | awk -F= '/^active=/{print \$2}'"
+          args:
+            executable: /bin/bash
+          register: dev2_active_before
+          changed_when: false
+          failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
+
+        - name: Determine new active value for DEV2
+          when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
+          ansible.builtin.set_fact:
+            dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ dev2_new_active }}
+          when: dev2_new_active is defined
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "fw_setenv active {{ dev2_new_active }}"
+          args:
+            executable: /bin/bash
+          register: dev2_setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped on DEV2
+          when: dev2_new_active is defined
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "fw_printenv active | awk -F= '/^active=/{print \$2}'"
+          args:
+            executable: /bin/bash
+          register: dev2_active_after
+          changed_when: false
+          failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
+
+        - name: Create specific prep marker on DEV2 (no generic)
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "touch '{{ _marker_specific }}'"
+          args:
+            executable: /bin/bash
+          register: dev2_marker_write
+          changed_when: true
+          ignore_errors: true
+
+    # ---------------------------- Reboot scheduling (normalized) ----------------------------
+    - name: Schedule delayed reboot on DEV2 minutes
+      when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        SECS="$(( {{ _reboot_minutes | int }} * 60 ))"
+        CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}"
+      args:
+        executable: /bin/bash
+      register: dev2_reboot_sched
+      changed_when: true
+      ignore_errors: true
+
+    - name: Note reboot request bad format
+      when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
+      ansible.builtin.debug:
+        msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"
+
+    # ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
+    - name: Build control queue payload for indoor updated journal
+      ansible.builtin.set_fact:
+        _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}"
+      when: not (_blocked | default(false))
+      delegate_to: localhost
+
+    - name: Build indoor updated journal payload text
+      ansible.builtin.set_fact:
+        journal_indoor_updated:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: Dev2 updated, scheduling reboot {{
+              'now' if (_reboot_requested and (_reboot_minutes | int) == 0)
+              else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0)
+              else 'not requested'
+            }}.
+            write_done={{ _write_success }},
+            active_before={{ (dev2_active_before.stdout | default('NA')) | trim }},
+            active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}
+      when: not (_blocked | default(false)) and (_write_success | bool)
+      delegate_to: localhost
+
+    - name: Publish indoor updated journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_updated | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_updated_resp
+      changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_updated_resp.status != 200) or
+        (rmq_journal_indoor_updated_resp.json is not defined) or
+        (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
+      when: journal_indoor_updated is defined
+      delegate_to: localhost
+
+    # ---------------------------- summary ----------------------------
+    - name: Summary show key results
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_passfile_used={{ dev2_passfile_used }}"
+          - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+          - "local_image_present={{ local_img.stat.exists | default(false) }}"
+          - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
+          - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
+          - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
+          - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}"
+          - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
+          - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
+          - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
+          - "reboot_requested={{ _reboot_requested | default(false) }}"
+          - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}"
+          - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}"
+          - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
+          - "prep_blocked={{ _prep_blocked | default(false) }}"
+          - "blocked={{ _blocked | default(false) }}"
+          - "journal={{ (_journal | default([])) | join(' || ') }}"
+
+  post_tasks:
+    - name: Cleanup always
+      block:
+        - ansible.builtin.debug:
+            msg: "entering cleanup block"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel best effort
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir best effort
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          changed_when: del_ip.rc == 0
+          failed_when: >
+            del_ip.rc != 0
+            and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+            and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
+
+    - name: Debug del ip results
+      ansible.builtin.debug:
+        msg:
+          - "del_ip.rc={{ del_ip.rc | default('') }}"
+          - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
+          - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
+      when: del_ip is defined

files/ansible-playbooks/update-reboot.yml

@@ -0,0 +1,4 @@
+---
+# update-reboot.yml — thin wrapper, no var forwarding. 
+# Delegates entirely to the unified updater.
+- import_playbook: update-rebootin222.yml

files/ansible-playbooks/update-rebootin222.yml

@@ -0,0 +1,831 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.2-r9778.bin
+    firmware_sha256: "15bc6f3492321196bb5220014c4441a4dceab2af5f570c46a23140cd37b65a02"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.2-r9778.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: 300
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost

files/ansible-playbooks/upgrade-confirmed.yml

@@ -0,0 +1,4 @@
+- import_playbook: update-rebootin222.yml
+  vars:
+    rebootin: 0
+

files/ansible-playbooks/uptime.yml

@@ -0,0 +1,13 @@
+---
+- name: Run uptime on a host
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Check uptime
+      ansible.builtin.raw: uptime
+      register: uptime_out
+
+    - name: Show result
+      ansible.builtin.debug:
+        var: uptime_out.stdout

files/ansible-playbooks/wifidebug14.yml

@@ -0,0 +1,384 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (from env with defaults)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('amq.default', true) }}"
+    queue2_name: "{{ lookup('env','QUEUE2') | default('queue2', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crontabs -f -l 8' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crontab", "crond -c /etc/crontabs -f -l 8")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            - name: Trigger full restart (system-stop; system-start) in background
+              when: do_edit
+              raw: |
+                 sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start
+              ignore_errors: true
+              changed_when: true
+
+            - name: Short grace delay before waiting
+              when: do_edit
+              delegate_to: localhost
+              pause:
+                seconds: 3
+
+            - name: Try to observe SSH port stopping (best effort)
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: stopped
+                timeout: 30
+                sleep: 2
+              ignore_errors: true
+
+            - name: Wait for SSH port to be accepting connections
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: started
+                timeout: 300
+                sleep: 2
+
+            - name: Reset Ansible SSH connection
+              when: do_edit
+              meta: reset_connection
+
+            - name: Verify command execution after restart (no python)
+              when: do_edit
+              raw: "echo rebooted_ok"
+              register: post_restart_probe
+              retries: 120
+              delay: 2
+              until: post_restart_probe is succeeded
+              changed_when: false
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        - name: Build journal_add payload
+          set_fact:
+            journal_payload_obj:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: "{{ result_status }}"
+
+        - name: Publish journal_add to queue2
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ queue2_name }}"
+              payload: "{{ journal_payload_obj | to_json }}"
+              payload_encoding: "string"
+          delegate_to: localhost
+
+        - name: Build deploy_wifidebug success payload
+          when: result_status == 'SUCCESS_DEPLOYED'
+          set_fact:
+            success_payload_obj:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "success"
+
+        - name: Publish deploy_wifidebug success to queue2
+          when: result_status == 'SUCCESS_DEPLOYED'
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ queue2_name }}"
+              payload: "{{ success_payload_obj | to_json }}"
+              payload_encoding: "string"
+          delegate_to: localhost
+
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"
+

files/ansible-playbooks/wifidebug15.yml

@@ -0,0 +1,447 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v15"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            - name: Trigger full restart (system-stop; system-start) in background
+              when: do_edit
+              raw: |
+                 sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start
+              ignore_errors: true
+              changed_when: true
+
+            - name: Short grace delay before waiting
+              when: do_edit
+              delegate_to: localhost
+              pause:
+                seconds: 3
+
+            - name: Try to observe SSH port stopping (best effort)
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: stopped
+                timeout: 30
+                sleep: 2
+              ignore_errors: true
+
+            - name: Wait for SSH port to be accepting connections
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: started
+                timeout: 300
+                sleep: 2
+
+            - name: Reset Ansible SSH connection
+              when: do_edit
+              meta: reset_connection
+
+            - name: Verify command execution after restart (no python)
+              when: do_edit
+              raw: "echo rebooted_ok"
+              register: post_restart_probe
+              retries: 120
+              delay: 2
+              until: post_restart_probe is succeeded
+              changed_when: false
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/wifidebug16.yml

@@ -0,0 +1,447 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v16"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            - name: Trigger full restart (system-stop; system-start) in background
+              when: do_edit
+              raw: |
+                 sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start
+              ignore_errors: true
+              changed_when: true
+
+            - name: Short grace delay before waiting
+              when: do_edit
+              delegate_to: localhost
+              pause:
+                seconds: 3
+
+            - name: Try to observe SSH port stopping (best effort)
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: stopped
+                timeout: 30
+                sleep: 2
+              ignore_errors: true
+
+            - name: Wait for SSH port to be accepting connections
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: started
+                timeout: 300
+                sleep: 2
+
+            - name: Reset Ansible SSH connection
+              when: do_edit
+              meta: reset_connection
+
+            - name: Verify command execution after restart (no python)
+              when: do_edit
+              raw: "echo rebooted_ok"
+              register: post_restart_probe
+              retries: 120
+              delay: 2
+              until: post_restart_probe is succeeded
+              changed_when: false
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/wifidebug17.yml

@@ -0,0 +1,476 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v17"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            # ===== ADDED: pre-download MD5 on device to guard download integrity =====
+            - name: MD5 of remote /tmp/config.json (pre-download)
+              when: do_edit
+              raw: "md5sum /tmp/config.json || busybox md5sum /tmp/config.json"
+              register: md5_remote_src
+              changed_when: false
+            # ===== END ADDED =====
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            # ===== ADDED: verify download integrity by comparing MD5s =====
+            - name: MD5 of local downloaded config.json
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json"
+              register: md5_local_src
+              changed_when: false
+
+            - name: Fail if MD5 mismatch on downloaded config.json
+              when:
+                - do_edit
+                - (md5_local_src.stdout.split()[0]) != (md5_remote_src.stdout.split()[0] if (md5_remote_src.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch on downloaded /tmp/config.json — aborting (download integrity check failed)."
+            # ===== END ADDED =====
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            # ===== ADDED: controller-side JSON sanity check (jq) =====
+            - name: Controller JSON sanity check (jq)
+              when: do_edit
+              delegate_to: localhost
+              shell: jq empty "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+              register: jq_local
+              changed_when: false
+              failed_when: jq_local.rc != 0
+            # ===== END ADDED =====
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
+
+            - name: Trigger full restart (system-stop; system-start) from controller with 10s cap
+              when: do_edit
+              delegate_to: localhost
+              shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+              args:
+                executable: /bin/bash
+              register: restart_kick
+              changed_when: true
+              failed_when: false
+
+
+            # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+            - name: Probe SSH with nc (24 tries, 5s each)
+              when: do_edit
+              delegate_to: localhost
+              shell: "nc -z -w 3 {{ ansible_host }} 22"
+              register: nc_probe
+              retries: 24
+              delay: 5
+              until: nc_probe.rc == 0
+              changed_when: false
+              failed_when: false
+
+            - name: Set ssh_up fact (from nc probe)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+            - name: Fail if SSH did not return after restart
+              when:
+                - do_edit
+                - not ssh_up | bool
+              fail:
+                msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
+
+            # ======== END OF SURGICAL CHANGE ========
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/wifidebug17.yml-old

@@ -0,0 +1,441 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v17"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
+
+            - name: Trigger full restart (system-stop; system-start) from controller with 10s cap
+              when: do_edit
+              delegate_to: localhost
+              shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+              args:
+                executable: /bin/bash
+              register: restart_kick
+              changed_when: true
+              failed_when: false
+
+
+            # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+            - name: Probe SSH with nc (24 tries, 5s each)
+              when: do_edit
+              delegate_to: localhost
+              shell: "nc -z -w 3 {{ ansible_host }} 22"
+              register: nc_probe
+              retries: 24
+              delay: 5
+              until: nc_probe.rc == 0
+              changed_when: false
+              failed_when: false
+
+            - name: Set ssh_up fact (from nc probe)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+            - name: Fail if SSH did not return after restart
+              when:
+                - do_edit
+                - not ssh_up | bool
+              fail:
+                msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
+
+            # ======== END OF SURGICAL CHANGE ========
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"
+