first commit

files/ansible-playbooks/afterupgrade_check.yml

@@ -0,0 +1,355 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_check.yml-from_chat

@@ -0,0 +1,355 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          expiration: "{{ (next_delay_ms | int) | string }}"
+          correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_check.yml-from_container

@@ -0,0 +1,432 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}'
+            Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag ---
+
+    - name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801")
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        indoor_fwver_norm: >-
+          {{
+            ( (dev2_fwver.stdout | default('') | trim)
+              | regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1')
+              | regex_replace('\\s+', ' ')
+              | trim
+            )
+          }}
+
+    - name: Publish NetBox custom field indoor_fwver to control queue
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fwver_norm | default(''))
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cf_indoor_fwver
+      changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool)
+
+    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_remove_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_remove"
+          task_result: "indoor-restart-scheduled"
+
+    - name: Publish indoor-restart-scheduled tag removal
+      when: tag_remove_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_remove_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_remove_sched
+      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
+
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host
+

files/ansible-playbooks/afterupgrade_check.yml-screewed

@@ -0,0 +1,359 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Journaling paths ----
+
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}'
+            Target='{{ target_version_full }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_indoor_check.yml

@@ -0,0 +1,744 @@
+---
+- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # ---------------- BusyBox-safe path prefix ----------------
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # ---------------- DEV1 (outer device) ----------------
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    dev1_iface: "br-wan"
+
+    # ---------------- DEV2 (indoor behind DEV1) ----------------
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ---- Normalize metadata (from delayed message) ----
+    - name: Normalize after-upgrade metadata
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        target_version_full: "{{ target_version | default('') }}"
+
+    # (NEW) Show what we received from the scheduler (for easy troubleshooting)
+    - name: Debug received scheduler metadata
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
+    - name: Check if TCP/22 on DEV1 is reachable
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # ---- If TCP down: journal + schedule next try or give up ----
+    - name: Build failure journal (TCP unreachable) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP 22 unreachable. Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (TCP unreachable)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    - name: Compute next-attempt delay (10 minutes) and counters
+      when: (_needs_retry | default(false)) | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: 600000
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path=TCP.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    - name: Build delayed after-upgrade payload for next attempt (10m)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after scheduling next attempt
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host
+
+
+    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname
+      when: nc_probe.rc == 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 hostname mismatch
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+    # ---------------- Step 2: Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 (ignore if exists)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0 and
+        ('File exists' not in (add_ip.stdout | default(''))) and
+        ('File exists' not in (add_ip.stderr | default('')))
+
+    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        for i in $(seq 1 25); do
+          p="$(shuf -i 20000-39999 -n1)"
+          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
+        done
+        exit 1
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control dir
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Start SSH ControlMaster tunnel via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
+          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
+    - name: Small delay for tunnel to settle
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        timeout: 1
+      changed_when: false
+
+    - name: Verify tunnel master running (ssh -O check)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
+      register: tun_check
+      changed_when: false
+
+    - name: Debug tunnel check
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tunnel_check.rc={{ tun_check.rc }}"
+          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
+
+    - name: Sanity confirm tunnel TCP reachability to DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug reachability result
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "nc.rc={{ nc_probe.rc }}"
+          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
+          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
+
+    - name: Stop if tunnel TCP check failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    # ---------------- Step 4: Determine working password for DEV2 ----------------
+    - name: Try both passfiles for DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
+            echo "$f"; exit 0;
+          fi
+        done
+        echo NONE
+      register: dev2_passfile_try
+      changed_when: false
+
+    - name: Save selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
+      changed_when: false
+
+    - name: Debug selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "dev2_passfile_used={{ dev2_passfile_used }}"
+
+    - name: Stop if no valid passfile found
+      ansible.builtin.meta: end_host
+      when: dev2_passfile_used == "NONE"
+
+    # ---------------- Step 5: Read firmware version on DEV2 ----------------
+    - name: Read firmware version from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
+      register: dev2_fwver
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show firmware version readout
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    # ---------------- Retry metadata + success evaluation (read OK) ----------------
+    - name: Normalize retry metadata for indoor checker
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+
+    - name: Check if firmware read succeeded (read_ok)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
+                    and ((dev2_fwver.stdout | default('') | trim) | length > 0)
+                    and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
+
+    # ===================== (NEW) Unconditional normalization + comparison =====================
+    # Compute expected_norm from target_version_full. If the full filename-like string is sent,
+    # we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
+
+    - name: Normalize expected target string (step 1: compute components)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
+        expected_norm_core: >-
+          {{
+            (target_version_full | default('') |
+              regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1'))
+            | default('', true)
+          }}
+
+    - name: Normalize expected target string (step 2: choose core if present)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+
+    # Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801"
+    - name: Normalize banner/firmux string from DEV2
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
+        banner_norm: >-
+          {{
+            (banner_raw | lower is search('-r[0-9]+$'))
+            | ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')))
+          }}
+
+    - name: Evaluate version match (normalized equality or contains)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        version_match: >-
+          {{
+            (expected_norm | length > 0)
+            and (
+              (banner_norm == expected_norm)
+              or (banner_norm is search(expected_norm))
+              or (expected_norm is search(banner_norm))
+            )
+          }}
+
+    - name: Debug compare snapshot (expected vs actual normalized)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm='{{ expected_norm }}'"
+          - "banner_norm='{{ banner_norm }}'"
+          - "version_match={{ version_match | default(false) }}"
+
+    # ===================== Journaling/Tagging paths =====================
+    # SUCCESS: read_ok AND version_match
+    - name: Build success journal payload
+      when: (read_ok | bool) and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ banner_raw }}' Target='{{ expected_norm }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_success
+      changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
+
+    - name: Build payload to add indoor-update-success tag (DEV1)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "indoor-update-success"
+
+    - name: Publish indoor-update-success tag
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add_success
+      changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
+
+    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_remove_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_remove"
+          task_result: "indoor-restart-scheduled"
+
+    - name: Publish indoor-restart-scheduled tag removal
+      when: tag_remove_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_remove_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_remove_sched
+      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
+
+    # --- Normalize firmware string and set custom field on success ---
+    - name: Capture raw firmware banner from DEV2 (for normalization)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_banner_raw: "{{ banner_raw }}"
+
+    - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
+      when:
+        - journal_success_payload is defined
+        - (fw_banner_raw | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_norm: >-
+          {{
+            fw_banner_raw
+            if (fw_banner_raw | lower is search('-r[0-9]+$'))
+            else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
+          }}
+
+    - name: Debug normalized firmware (indoor_fwver)
+      when: fw_norm is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
+
+    - name: Publish custom_field_set indoor_fwver
+      when:
+        - fw_norm is defined
+        - (fw_norm | length) > 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': fw_norm
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_customfield_fw
+      changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
+
+    # MISMATCH path: firmware readable but does NOT match expected target
+    - name: Build mismatch journal payload
+      when: (read_ok | bool) and (not (version_match | bool))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    - name: Stop host after mismatch evaluation
+      when: journal_mismatch_payload is defined
+      ansible.builtin.meta: end_host
+
+    # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
+    - name: Build failure journal payload (indoor firmware read failed)
+      when: not (read_ok | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish failure journal to control queue
+      when: journal_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_fail
+      changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
+
+    - name: Compute retry parameters (10 minutes)
+      when: not (read_ok | bool)
+      ansible.builtin.set_fact:
+        next_attempt: "{{ (attempt | int) + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: "{{ 600000 }}"
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_gaveup
+      changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
+
+    # Only schedule next attempt if we still have budget left
+    - name: Build delayed payload for next indoor attempt (10 min)
+      when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id | default('') }}"
+          original_emitted_at: "{{ original_emitted_at | default('') }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id | default('') }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+  post_tasks:
+    - name: Cleanup tunnel and temp IP
+      block:
+        - ansible.builtin.debug:
+            msg: "Cleaning up tunnel + temp IP"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP from DEV1
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          failed_when: false
+          changed_when: false

files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries

@@ -0,0 +1,201 @@
+---
+- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # ---------------- BusyBox-safe path prefix ----------------
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # ---------------- DEV1 (outer device) ----------------
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    dev1_iface: "br-wan"
+
+    # ---------------- DEV2 (indoor behind DEV1) ----------------
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 hostname mismatch
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+    # ---------------- Step 2: Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 (ignore if exists)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0 and
+        ('File exists' not in (add_ip.stdout | default(''))) and
+        ('File exists' not in (add_ip.stderr | default('')))
+
+    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        for i in $(seq 1 25); do
+          p="$(shuf -i 20000-39999 -n1)"
+          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
+        done
+        exit 1
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control dir
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+
+    - name: Start SSH ControlMaster tunnel via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
+          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    # ---------------- Step 4: Determine working password for DEV2 ----------------
+    - name: Try both passfiles for DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
+            echo "$f"; exit 0;
+          fi
+        done
+        echo NONE
+      register: dev2_passfile_try
+      changed_when: false
+
+    - name: Save selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
+      changed_when: false
+
+    - name: Stop if no valid passfile found
+      ansible.builtin.meta: end_host
+      when: dev2_passfile_used == "NONE"
+
+    # ---------------- Step 5: Read firmware version on DEV2 ----------------
+    - name: Read firmware version from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
+      register: dev2_fwver
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show firmware version readout
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    # ---------------- Step 6: Publish result journal ----------------
+    - name: Build journal payload
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
+            tunnel_port={{ _local_port }}
+
+    - name: Publish journal to control queue
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_fwver
+      changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool)
+
+  post_tasks:
+    - name: Cleanup tunnel and temp IP
+      block:
+        - ansible.builtin.debug:
+            msg: "Cleaning up tunnel + temp IP"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP from DEV1
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          failed_when: false
+          changed_when: false

files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors

@@ -0,0 +1,644 @@
+---
+- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # ---------------- BusyBox-safe path prefix ----------------
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # ---------------- DEV1 (outer device) ----------------
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    dev1_iface: "br-wan"
+
+    # ---------------- DEV2 (indoor behind DEV1) ----------------
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    # ---- Normalize metadata (from delayed message) ----
+    - name: Normalize after-upgrade metadata
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        target_version_full: "{{ target_version | default('') }}"
+
+    # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
+    - name: Check if TCP/22 on DEV1 is reachable
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # ---- If TCP down: journal + schedule next try or give up ----
+    - name: Build failure journal (TCP unreachable) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            TCP 22 unreachable. Correlation={{ correlation_id }}
+            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+        _needs_retry: true
+
+    - name: Publish failure journal (TCP unreachable)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    - name: Compute next-attempt delay (10 minutes) and counters
+      when: (_needs_retry | default(false)) | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: 600000
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path=TCP.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    - name: Build delayed after-upgrade payload for next attempt (10m)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after scheduling next attempt
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host
+
+
+    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname
+      when: nc_probe.rc == 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 hostname mismatch
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+    # ---------------- Step 2: Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 (ignore if exists)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0 and
+        ('File exists' not in (add_ip.stdout | default(''))) and
+        ('File exists' not in (add_ip.stderr | default('')))
+
+    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        for i in $(seq 1 25); do
+          p="$(shuf -i 20000-39999 -n1)"
+          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
+        done
+        exit 1
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control dir
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Start SSH ControlMaster tunnel via DEV1
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p '{{ dev1_pass }}' ssh -f -N \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
+          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
+    - name: Small delay for tunnel to settle
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        timeout: 1
+      changed_when: false
+
+    - name: Verify tunnel master running (ssh -O check)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
+      register: tun_check
+      changed_when: false
+
+    - name: Debug tunnel check
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tunnel_check.rc={{ tun_check.rc }}"
+          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
+
+    - name: Sanity confirm tunnel TCP reachability to DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug reachability result
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "nc.rc={{ nc_probe.rc }}"
+          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
+          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
+
+    - name: Stop if tunnel TCP check failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    # ---------------- Step 4: Determine working password for DEV2 ----------------
+    - name: Try both passfiles for DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
+            echo "$f"; exit 0;
+          fi
+        done
+        echo NONE
+      register: dev2_passfile_try
+      changed_when: false
+
+    - name: Save selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
+      changed_when: false
+
+    - name: Debug selected DEV2 passfile
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "dev2_passfile_used={{ dev2_passfile_used }}"
+
+    - name: Stop if no valid passfile found
+      ansible.builtin.meta: end_host
+      when: dev2_passfile_used == "NONE"
+
+    # ---------------- Step 5: Read firmware version on DEV2 ----------------
+    - name: Read firmware version from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
+      register: dev2_fwver
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show firmware version readout
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    # ---------------- Retry metadata + success evaluation ----------------
+    - name: Normalize retry metadata for indoor checker
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+
+    - name: Evaluate indoor success (firmware read OK)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        indoor_success: "{{ (dev2_fwver.rc | default(1) == 0)
+                            and ((dev2_fwver.stdout | default('') | trim) | length > 0)
+                            and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
+
+    # ---------------- Step 6: Publish result journal (+ tags on success) ----------------
+    # SUCCESS PATH: add tag, remove scheduled tag, and journal with "success"
+    - name: Build success journal payload
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
+            tunnel_port={{ _local_port }}
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_success
+      changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
+
+    - name: Build payload to add indoor-update-success tag (DEV1)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "indoor-update-success"
+
+    - name: Publish indoor-update-success tag
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add_success
+      changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
+
+    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_remove_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_remove"
+          task_result: "indoor-restart-scheduled"
+
+    - name: Publish indoor-restart-scheduled tag removal
+      when: tag_remove_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_remove_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_remove_sched
+      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
+
+    # --- Normalize firmware string and set custom field on success ---
+    - name: Capture raw firmware banner from DEV2 (for normalization)
+      when: indoor_success | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}"
+
+    - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
+      when:
+        - indoor_success | bool
+        - (fw_banner_raw | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fw_norm: >-
+          {{
+            fw_banner_raw
+            if (fw_banner_raw | lower is search('-r[0-9]+$'))
+            else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
+          }}
+
+    - name: Debug normalized firmware (indoor_fwver)
+      when: fw_norm is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
+
+    - name: Publish custom_field_set indoor_fwver
+      when:
+        - indoor_success | bool
+        - fw_norm is defined
+        - (fw_norm | length) > 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': fw_norm
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_customfield_fw
+      changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
+
+    # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
+    - name: Build failure journal payload (indoor firmware read failed)
+      when: not (indoor_success | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish failure journal to control queue
+      when: journal_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_fail
+      changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
+
+    - name: Compute retry parameters (10 minutes)
+      when: not (indoor_success | bool)
+      ansible.builtin.set_fact:
+        next_attempt: "{{ (attempt | int) + 1 }}"
+        next_delay_sec: 600
+        next_delay_ms: "{{ 600000 }}"
+
+    - name: Build final gave-up journal (max attempts reached)
+      when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
+            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_gaveup
+      changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
+
+    # Only schedule next attempt if we still have budget left
+    - name: Build delayed payload for next indoor attempt (10 min)
+      when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int))
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_indoor_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id | default('') }}"
+          original_emitted_at: "{{ original_emitted_at | default('') }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id | default('') }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+  post_tasks:
+    - name: Cleanup tunnel and temp IP
+      block:
+        - ansible.builtin.debug:
+            msg: "Cleaning up tunnel + temp IP"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP from DEV1
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          failed_when: false
+          changed_when: false
+

files/ansible-playbooks/basicpass

@@ -0,0 +1 @@
+wavewave

files/ansible-playbooks/basicpass2

@@ -0,0 +1 @@
+admin

files/ansible-playbooks/checkversioncli.yml

@@ -0,0 +1,18 @@
+---
+- name: Simple script to show current version
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: grab current version
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        cat /etc/banner | grep 2
+      register: banner_out
+      changed_when: true
+
+    - name: show the current version
+      ansible.builtin.debug:
+        msg: "{{ banner_out.stdout | trim }}"
+

files/ansible-playbooks/deploy-scroll24.yml

@@ -0,0 +1,277 @@
+---
+- name: Deploy scroll24 script and cron configuration
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # Version tag for scroll24 deployment (operator-controlled)
+    scroll24_version: "v1.1"
+
+    # Controller-side source paths (same place as wifidebug.sh)
+    src_script: "files/scroll24.sh"
+    src_cron_snippet: "files/crond-root-scroll24"
+
+    # Remote destinations
+    remote_script: "/root/scroll24.sh"
+    remote_crontab: "/etc/crontabs/root"
+    tmp_cron_snippet: "/tmp/crond-root-scroll24.snippet"
+    tmp_cron_new: "/tmp/cron.root.new"
+    crontab_backup_dir: "/etc/crontabs"
+
+    # RabbitMQ (same contract/style as your other playbooks)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    ##########################################################################
+    # 0. Controller sanity check for tools
+    ##########################################################################
+    - name: Verify controller tools (sshpass, sha256sum/busybox)
+      delegate_to: localhost
+      shell: |
+        command -v sshpass >/dev/null 2>&1 \
+        && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1)
+      args: { executable: /bin/bash }
+      register: ctrl_tools
+      changed_when: false
+      failed_when: ctrl_tools.rc != 0
+
+    ##########################################################################
+    # 1. Upload /root/scroll24.sh if changed (sha256 verified)
+    ##########################################################################
+    - name: Compute local sha256 of scroll24.sh
+      delegate_to: localhost
+      command: sha256sum {{ src_script }}
+      register: sha_local
+      changed_when: false
+
+    - name: Extract local sha256 digest (regex)
+      delegate_to: localhost
+      set_fact:
+        local_hash: "{{ (sha_local.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}"
+
+    - name: Compute remote sha256 of /root/scroll24.sh
+      raw: "sha256sum {{ remote_script }} 2>/dev/null || busybox sha256sum {{ remote_script }} 2>/dev/null || true"
+      register: sha_remote
+      changed_when: false
+      failed_when: false
+
+    - name: Extract remote sha256 digest (regex)
+      set_fact:
+        remote_hash: "{{ (sha_remote.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}"
+
+    - name: Decide if scroll24.sh needs upload
+      set_fact:
+        script_changed: "{{ (remote_hash | length == 0) or (local_hash != remote_hash) }}"
+
+    - name: Upload scroll24.sh via scp (if needed)
+      when: script_changed | bool
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ src_script }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_script }}
+      register: scp_scroll24
+      changed_when: true
+
+    - name: Ensure /root/scroll24.sh permissions and ownership
+      raw: "chown root:root {{ remote_script }} && chmod 0755 {{ remote_script }}"
+      changed_when: script_changed | bool
+
+    - name: Recompute remote sha256 after upload
+      when: script_changed | bool
+      raw: "sha256sum {{ remote_script }} || busybox sha256sum {{ remote_script }}"
+      register: sha_remote_after
+      changed_when: false
+      failed_when: false
+
+    - name: Extract remote-after sha256 digest (regex)
+      when: script_changed | bool
+      set_fact:
+        remote_hash_after: "{{ (sha_remote_after.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}"
+
+    - name: Fail if scroll24.sh sha256 mismatch after upload
+      when: script_changed | bool and (remote_hash_after | default('')) != (local_hash | default(''))
+      fail:
+        msg: "sha256 mismatch between controller and remote scroll24.sh"
+
+    ##########################################################################
+    # 1b. Extract the period (minutes) from the first line of the snippet
+    ##########################################################################
+    - name: Extract period minutes from first snippet line (*/N …)
+      delegate_to: localhost
+      shell: |
+        awk 'NR==1{
+          f=$1;
+          if (f ~ /^\*\/[0-9]+$/) { gsub("^\\*/","",f); print f; exit }
+          else if (f ~ /^[0-9]+$/) { print f; exit }
+          else { print ""; exit }
+        }' {{ src_cron_snippet }}
+      args: { executable: /bin/bash }
+      register: cron_period_cmd
+      changed_when: false
+      failed_when: false
+
+    - name: Set cron_period fact
+      delegate_to: localhost
+      set_fact:
+        cron_period: "{{ (cron_period_cmd.stdout | trim) }}"
+
+    ##########################################################################
+    # 2. Upload cron snippet to device /tmp (operator-provided content)
+    ##########################################################################
+    - name: Push cron snippet to device tmp
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ src_cron_snippet }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ tmp_cron_snippet }}
+      register: scp_snippet
+      changed_when: true
+
+    ##########################################################################
+    # 3. Crontab subset logic (only scroll24.sh lines)
+    ##########################################################################
+    - name: Ensure /etc/crontabs/root exists (with perms)
+      raw: |
+        if [ ! -f {{ remote_crontab }} ]; then
+          touch {{ remote_crontab }};
+        fi
+        chown root:root {{ remote_crontab }};
+        chmod 0644 {{ remote_crontab }};
+      changed_when: false
+
+    - name: Extract current scroll24 lines from crontab
+      raw: "grep -F 'scroll24.sh' {{ remote_crontab }} || true"
+      register: cron_subset
+      changed_when: false
+      failed_when: false
+
+    - name: Normalize controller snippet (for compare only)
+      delegate_to: localhost
+      shell: |
+        grep -v '^[[:space:]]*$' {{ src_cron_snippet }} \
+        | sed 's/[[:space:]]\+/ /g' | sed 's/[[:space:]]*$//' | sort -u
+      register: norm_snippet
+      changed_when: false
+
+    - name: Normalize device scroll24 subset (for compare only)
+      delegate_to: localhost
+      shell: |
+        printf "%s\n" "{{ cron_subset.stdout | default('') }}" \
+        | grep -v '^[[:space:]]*$' \
+        | sed 's/[[:space:]]\+/ /g' | sed 's/[[:space:]]*$//' | sort -u
+      register: norm_remote_subset
+      changed_when: false
+
+    - name: Decide if crontab needs update (subset compare)
+      delegate_to: localhost
+      set_fact:
+        crontab_changed: "{{ (norm_snippet.stdout | trim) != (norm_remote_subset.stdout | trim) }}"
+
+    - name: Backup current crontab (timestamped)
+      when: crontab_changed | bool
+      raw: "cp -a {{ remote_crontab }} {{ crontab_backup_dir }}/root.bak.$(date +%Y%m%d%H%M%S)"
+      changed_when: true
+
+    - name: Replace scroll24 lines in crontab (preserve all others; tidy splice)
+      when: crontab_changed | bool
+      raw: "grep -v 'scroll24\\.sh' {{ remote_crontab }} > {{ tmp_cron_new }} && awk 'BEGIN{for(i=1;i<=NR;i++)a[i]=$0} {a[NR]=$0} END{e=NR; while(e>0 && a[e] ~ /^[[:space:]]*$/){e--}; for(i=1;i<=e;i++) print a[i]}' {{ tmp_cron_new }} > {{ tmp_cron_new }}.trim && mv {{ tmp_cron_new }}.trim {{ tmp_cron_new }} && cat {{ tmp_cron_snippet }} >> {{ tmp_cron_new }} && printf '\\n' >> {{ tmp_cron_new }} && mv {{ tmp_cron_new }} {{ remote_crontab }} && chown root:root {{ remote_crontab }} && chmod 0644 {{ remote_crontab }}"
+      changed_when: true
+
+    ##########################################################################
+    # 4. Restart crond if script or cron changed (with :51–:59 guard)
+    ##########################################################################
+    - name: Check if restart required
+      set_fact:
+        need_restart: "{{ (script_changed | bool) or (crontab_changed | bool) }}"
+
+    - name: Get current seconds
+      when: need_restart | bool
+      raw: "date +%S"
+      register: nowsec
+      changed_when: false
+
+    - name: Sleep 10s if seconds 51-59
+      when: need_restart | bool and (nowsec.stdout | int >= 51)
+      pause:
+        seconds: 10
+
+    - name: Restart crond via move/move
+      when: need_restart | bool
+      raw: "mv /tmp/launchd/services/crond /root/crond && sleep 1 && mv /root/crond /tmp/launchd/services/crond"
+      register: crond_restart
+      changed_when: true
+      failed_when: false
+
+    - name: Verify crond is running
+      when: need_restart | bool
+      raw: "pgrep -f '/usr/sbin/crond' || busybox pgrep crond || echo missing"
+      register: crond_pid
+      changed_when: false
+      failed_when: false
+
+    ##########################################################################
+    # 5. Final journals to RabbitMQ (controller)
+    ##########################################################################
+    - name: Publish final scroll24 deployment journal
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'scroll24: '
+              ~ (script_changed | ternary('script installed/updated; ', 'script up-to-date; '))
+              ~ (crontab_changed | ternary('cron updated; ', 'cron already matches; '))
+              ~ (need_restart | ternary('crond restarted; ', 'crond unchanged; '))
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Set NetBox custom field scroll24 -> "<version>_<period>"
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'scroll24',
+            'task_result': scroll24_version ~ '_' ~ (cron_period | default(''))
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+

files/ansible-playbooks/full-upgrade-check.yml

@@ -0,0 +1,20 @@
+- import_playbook: update-rebootin222.yml
+  vars:
+    rebootin: 0
+
+- name:
+  hosts: all                # will match your limited hosts
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Wait 5 minutes
+      ansible.builtin.wait_for:
+        timeout: "{{ pause_minutes | default(300) }}"
+      run_once: true
+      delegate_to: localhost
+      check_mode: no
+      tags: always
+
+- import_playbook: checkversioncli.yml
+#- import_playbook: setntptime.yml
+

files/ansible-playbooks/full-upgrade-wifi14.yml

@@ -0,0 +1,21 @@
+- import_playbook: update-rebootin.yml
+  vars:
+    rebootin: 0
+
+- name:
+  hosts: all                # will match your limited hosts
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Wait 5 minutes
+      ansible.builtin.wait_for:
+        timeout: "{{ pause_minutes | default(300) }}"
+      run_once: true
+      delegate_to: localhost
+      check_mode: no
+      tags: always
+
+- import_playbook: checkversioncli.yml
+- import_playbook: setntptime.yml
+- import_playbook: wifidebug14.yml
+- import_playbook: system-stop-system-start.yml

files/ansible-playbooks/multissidfix-changessid.yml

@@ -0,0 +1,826 @@
+---
+- name: "Multi-SSID fix (phase 1: persist + runtime apply)"
+  hosts: all
+  gather_facts: false
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (same contract as wifidebug16.yml)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    # Reboot parachute delay (seconds)
+    parachute_delay_s: 600
+
+    # Local working paths (controller)
+    local_cfg:  "/tmp/{{ inventory_hostname }}_config.json"
+    local_new:  "/tmp/{{ inventory_hostname }}_config.json.new"
+    local_step: "/tmp/{{ inventory_hostname }}_config.step"
+    local_diff: "/tmp/{{ inventory_hostname }}_config.diff"
+    local_dd:   "/tmp/{{ inventory_hostname }}_dd.json"
+
+    # External scripts/filters (next to wifidebug.sh in your repo)
+    dd_script_src: "files/wirelessduediligence.sh"
+    f_1vap0_src:   "files/filter_1vap_id0.jq"
+    f_1vap1_src:   "files/filter_1vap_id1.jq"
+    f_2_01_src:    "files/filter_2vaps_0_1.jq"
+    f_2_10_src:    "files/filter_2vaps_1_0.jq"
+
+    # Controller tmp destinations
+    dd_script: "/tmp/wirelessduediligence.sh"
+    f_1vap0:   "/tmp/filter_1vap_id0.jq"
+    f_1vap1:   "/tmp/filter_1vap_id1.jq"
+    f_2_01:    "/tmp/filter_2vaps_0_1.jq"
+    f_2_10:    "/tmp/filter_2vaps_1_0.jq"
+
+    # Remote (device)
+    remote_cfg:    "/tmp/config.json"
+    remote_new:    "/tmp/config.json.new"
+    remote_backup: "/tmp/config.json.multissidfix1.backup"
+
+    # Optional new SSID name (set via env NEW_SSID to trigger rename)
+    new_ssid_name: "{{ lookup('env','NEW_SSID') | default('ikeja R5 a day hotspot', true) }}"
+
+  tasks:
+
+    ##########################################################################
+    # a) Connectivity + basic sanity
+    ##########################################################################
+    - name: SSH reachability probe
+      raw: "echo ping"
+      register: ping_result
+      ignore_errors: true
+
+    - name: Stop if SSH unreachable (soft-fail)
+      when: ping_result is failed
+      block:
+        - name: Journal soft-fail (no SSH)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — SSH not reachable'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Ensure remote /tmp/config.json exists
+      raw: "test -s {{ remote_cfg }}"
+      register: remote_cfg_check
+      changed_when: false
+
+    - name: Stop if /tmp/config.json missing (soft-fail)
+      when: remote_cfg_check.rc != 0
+      block:
+        - name: Journal soft-fail (missing config.json)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — missing /tmp/config.json'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Check controller tools (jq/sshpass/diff/sha256sum)
+      delegate_to: localhost
+      shell: |
+        command -v jq >/dev/null 2>&1 \
+        && command -v sshpass >/dev/null 2>&1 \
+        && command -v diff >/dev/null 2>&1 \
+        && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1)
+      args: { executable: /bin/bash }
+      register: ctrl_tools
+      changed_when: false
+
+    - name: Stop if controller tools missing (soft-fail)
+      when: ctrl_tools.rc != 0
+      block:
+        - name: Journal soft-fail (missing tools)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — controller missing jq/sshpass/diff/sha256sum'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # b) Due diligence (external script, run on controller)
+    ##########################################################################
+    - name: Copy due diligence script to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ dd_script_src }}"
+        dest: "{{ dd_script }}"
+        mode: "0755"
+
+    - name: Pull /tmp/config.json from device to controller
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_cfg }}
+        {{ local_cfg }}
+
+    - name: Run due diligence on controller
+      delegate_to: localhost
+      shell: "sh {{ dd_script | quote }} {{ local_cfg | quote }} > {{ local_dd | quote }}"
+      args: { executable: /bin/sh }
+      changed_when: false
+
+    - name: Parse due diligence JSON
+      delegate_to: localhost
+      set_fact:
+        dd: "{{ lookup('file', local_dd) | from_json }}"
+
+    # Normalize types for robust `when:` checks
+    - name: Extract & normalize dd facts
+      delegate_to: localhost
+      set_fact:
+        w0_total_i: "{{ (dd.radios.wifi0.vaps_total | int) }}"
+        w1_total_i: "{{ (dd.radios.wifi1.vaps_total | int) }}"
+        w0_idx_i:   "{{ (dd.radios.wifi0.indices | map('int') | list) }}"
+        w1_idx_i:   "{{ (dd.radios.wifi1.indices | map('int') | list) }}"
+
+    ##########################################################################
+    # c) Soft-fail: layout policy checks (one journal)
+    ##########################################################################
+    - name: Init soft-fail reasons (policy)
+      delegate_to: localhost
+      set_fact:
+        softfail_reasons: []
+
+    - name: wifi0 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w0_total_i | int) == 1 and (0 not in w0_idx_i and 1 not in w0_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 ONE_VAP but index not 0/1 (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w0_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 MORE_THAN_TWO_VAPS (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w1_total_i | int) == 1 and (0 not in w1_idx_i and 1 not in w1_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 ONE_VAP but index not 0/1 (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w1_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 MORE_THAN_TWO_VAPS (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: Publish soft-fail journal & stop host (policy)
+      when: (softfail_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (policy)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (policy) — ' ~ (softfail_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # d) Filters to controller tmp + candidate generation (per radio)
+    ##########################################################################
+    - name: Copy jq filters to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        mode: "0644"
+      loop:
+        - { src: "{{ f_1vap0_src }}", dest: "{{ f_1vap0 }}" }
+        - { src: "{{ f_1vap1_src }}", dest: "{{ f_1vap1 }}" }
+        - { src: "{{ f_2_01_src }}",  dest: "{{ f_2_01 }}"  }
+        - { src: "{{ f_2_10_src }}",  dest: "{{ f_2_10 }}"  }
+
+    - name: Start from current config as working file
+      delegate_to: localhost
+      copy:
+        src: "{{ local_cfg }}"
+        dest: "{{ local_step }}"
+        mode: "0644"
+
+    # wifi0 choice
+    - name: wifi0 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (0 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (1 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i == [1,0])
+      changed_when: true
+
+    # wifi1 choice
+    - name: wifi1 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (0 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (1 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i == [1,0])
+      changed_when: true
+
+    # Optional SSID rename (only if new_ssid_name is defined)
+    - name: Set SSID for the single enabled AP VAP per radio (wifi0 & wifi1)
+      delegate_to: localhost
+      shell: |
+        jq --arg SSID "{{ new_ssid_name }}" '
+          .wireless.radios |=
+            with_entries(
+              .value.vaps =
+                ((.value.vaps // []) | map(
+                  if (.mode=="ap" and (.enabled==true)) then
+                    (.ssid = $SSID)
+                  else .
+                  end
+                ))
+            )
+        ' {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}
+      args: { executable: /bin/bash }
+      when: new_ssid_name is defined
+      changed_when: true
+
+    - name: Debug normalized types
+      delegate_to: localhost
+      debug:
+        msg:
+          - "w0_total_i(type)={{ w0_total_i | type_debug }}  value={{ w0_total_i }}"
+          - "w1_total_i(type)={{ w1_total_i | type_debug }}  value={{ w1_total_i }}"
+
+    - name: Move working file to final candidate
+      delegate_to: localhost
+      copy:
+        src: "{{ local_step }}"
+        dest: "{{ local_new }}"
+        mode: "0644"
+
+    ##########################################################################
+    # e) Candidate checks — accumulate reasons; soft-fail once if any
+    ##########################################################################
+    - name: Init soft-fail reasons (candidate)
+      delegate_to: localhost
+      set_fact:
+        cand_reasons: []
+
+    - name: Check JSON validity
+      delegate_to: localhost
+      shell: "jq -e '.' {{ local_new | quote }} >/dev/null"
+      args: { executable: /bin/bash }
+      register: json_valid
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if JSON invalid
+      delegate_to: localhost
+      when: json_valid.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'invalid JSON syntax in candidate' ] }}"
+
+    - name: Semantic — wifi0 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 semantic fails
+      delegate_to: localhost
+      when: sem_w0.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi1 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 semantic fails
+      delegate_to: localhost
+      when: sem_w1.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi0 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 lbd check fails
+      delegate_to: localhost
+      when: sem_w0_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Semantic — wifi1 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 lbd check fails
+      delegate_to: localhost
+      when: sem_w1_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Get original file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_cfg }}"
+      register: stat_old
+      changed_when: false
+
+    - name: Get new file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_new }}"
+      register: stat_new
+      changed_when: false
+
+    - name: Add reason if size delta > 2%
+      delegate_to: localhost
+      when: stat_old.stat.size | int == 0 or
+            ( ((stat_new.stat.size | int) - (stat_old.stat.size | int)) | abs ) > ( (stat_old.stat.size | int) * 0.02 )
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'candidate size delta >2% (old=' ~ (stat_old.stat.size | string) ~ ', new=' ~ (stat_new.stat.size | string) ~ ')' ] }}"
+
+    - name: Build unified diff (first 200 lines)
+      delegate_to: localhost
+      shell: "diff -u {{ local_cfg | quote }} {{ local_new | quote }} | sed -n '1,200p' > {{ local_diff | quote }} || true"
+      args: { executable: /bin/bash }
+      changed_when: false
+
+    - name: Show unified diff (first 200 lines)
+      delegate_to: localhost
+      debug:
+        msg: "{{ lookup('file', local_diff) | default('(no diff output)') }}"
+
+    - name: sha256 (controller) of candidate
+      delegate_to: localhost
+      command: sha256sum {{ local_new }}
+      register: sha_local
+      changed_when: false
+
+    - name: Copy candidate to device temp
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ local_new }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_new }}
+      register: scp_new
+      changed_when: true
+
+    - name: sha256 (remote) of candidate
+      raw: "sha256sum {{ remote_new }} || busybox sha256sum {{ remote_new }}"
+      register: sha_remote
+      changed_when: false
+
+    - name: Add reason if sha256 mismatch
+      delegate_to: localhost
+      when: (sha_local.stdout.split()[0]) != (sha_remote.stdout.split()[0])
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'sha256 mismatch controller vs remote' ] }}"
+
+    - name: Detect SSIDs with NBSP (informational)
+      delegate_to: localhost
+      shell: |
+        nbsp="$(printf '\302\240')"
+        jq -r '..|objects|.ssid? // empty' {{ local_new | quote }} | grep -F "$nbsp" || true
+      args: { executable: /bin/bash }
+      register: nbsp_lines
+      changed_when: false
+
+    - name: Save NBSP report lines
+      set_fact:
+        ssid_nbsp_lines: "{{ nbsp_lines.stdout_lines | default([]) }}"
+
+    - name: Publish soft-fail journal & stop host (candidate issues)
+      when: (cand_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (candidate)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (candidate) — ' ~ (cand_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # f) Backup, parachute, promote, persist, apply, outcome
+    ##########################################################################
+    - name: Backup current config on device
+      raw: "cp -a {{ remote_cfg }} {{ remote_backup }}"
+      changed_when: true
+
+    # --- PRE-APPLY HEADS-UP ---
+    - name: Journal — candidate validated, will promote/persist/apply shortly
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: candidate passed checks; proceeding to backup + parachute + promote + sysconf -w + system-stop/start. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Arm reboot parachute (BusyBox reboot -d {{ parachute_delay_s }})
+      raw: "sh -c 'trap \"\" HUP; reboot -d {{ parachute_delay_s }} </dev/null >/dev/null 2>&1 &'"
+      changed_when: true
+      ignore_errors: true
+
+    - name: Promote candidate to active config
+      raw: "mv {{ remote_new }} {{ remote_cfg }} && chown root:root {{ remote_cfg }} && chmod 0644 {{ remote_cfg }}"
+      changed_when: true
+
+    # IMPORTANT: persist BEFORE runtime restart
+    - name: Persist config to flash (sysconf -w)
+      raw: "sysconf -w"
+      register: sysconf_write
+      changed_when: true
+      failed_when: false
+
+    # --- PRE-RESTART COMMIT ---
+    - name: Pre-restart journal (promoted & persisted; about to restart from controller)
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: promoted candidate, persisted (sysconf -w rc=' ~ (sysconf_write.rc | default('n/a')) | string ~ '), '
+              ~ 'system-stop/start launching from controller with 10s cap; '
+              ~ 'parachute -d ' ~ parachute_delay_s|string ~ 's armed. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+              ~ ', nbsp_ssids=' ~ (ssid_nbsp_lines|length)|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Launch restart from controller with 10s cap (SSH command)
+      delegate_to: localhost
+      shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+      args:
+        executable: /bin/bash
+      register: restart_kick
+      changed_when: true
+      failed_when: false
+
+    - name: Note restart_kick result (debug)
+      delegate_to: localhost
+      debug:
+        msg:
+          - "restart_kick.rc={{ restart_kick.rc }}"
+          - "stdout(last 5 lines): {{ (restart_kick.stdout_lines | default([]))[-5:] | default([]) }}"
+          - "stderr(last 5 lines): {{ (restart_kick.stderr_lines | default([]))[-5:] | default([]) }}"
+
+    # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+    - name: Probe SSH with nc (24 tries, 5s each)
+      delegate_to: localhost
+      shell: "nc -z -w 3 {{ ansible_host }} 22"
+      register: nc_probe
+      retries: 24
+      delay: 5
+      until: nc_probe.rc == 0
+      changed_when: false
+      failed_when: false
+
+    - name: Set ssh_up fact
+      delegate_to: localhost
+      set_fact:
+        ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+    # --- SUCCESS PATH ---
+    - name: Post-return actions (only if SSH came back)
+      when: ssh_up | bool
+      block:
+
+        - name: Disarm delayed reboot if present
+          raw: "pgrep -x reboot && kill -9 $(pgrep -x reboot) || true"
+          changed_when: true
+          ignore_errors: true
+          register: disarm_reboot
+
+        # Re-run simple semantic checks on the active config
+        - name: Device semantic — wifi0 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi0 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: sha256 (remote) of current active config (post-return)
+          raw: "sha256sum {{ remote_cfg }} || busybox sha256sum {{ remote_cfg }}"
+          register: sha_remote_after
+          changed_when: false
+          failed_when: false
+
+        - name: Decide post-return checks summary (controller side)
+          delegate_to: localhost
+          set_fact:
+            multissid_post_summary:
+              reboot_disarmed: "{{ (disarm_reboot is defined) and (disarm_reboot.rc is defined) and (disarm_reboot.rc in [0]) }}"
+              persisted_ok: "{{ (sysconf_write.rc | default(0)) == 0 }}"
+              dev_sem_ok:
+                w0_one: "{{ (dev_sem_w0.rc | default(1)) == 0 }}"
+                w1_one: "{{ (dev_sem_w1.rc | default(1)) == 0 }}"
+                w0_lbd: "{{ (dev_sem_w0_lbd.rc | default(1)) == 0 }}"
+                w1_lbd: "{{ (dev_sem_w1_lbd.rc | default(1)) == 0 }}"
+              cfg_hash_match: "{{ (sha_remote_after.stdout.split()[0] | default('')) == (sha_remote.stdout.split()[0] | default('')) }}"
+              old_size: "{{ stat_old.stat.size | default('n/a') }}"
+              new_size: "{{ stat_new.stat.size | default('n/a') }}"
+              nbsp_count: "{{ (ssid_nbsp_lines | default([])) | length }}"
+
+        - name: Publish final success journal (device returned; persisted; checks pass)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result':
+                  (
+                    'multissidfix: success — device is back; '
+                    ~ (multissid_post_summary.reboot_disarmed | ternary('parachute disarmed; ', 'parachute was not armed; '))
+                    ~ (multissid_post_summary.persisted_ok | ternary('persisted (sysconf -w); ', 'persist failed; '))
+                    ~ 'semantics: '
+                    ~ 'w0_one=' ~ (multissid_post_summary.dev_sem_ok.w0_one | string) ~ ', '
+                    ~ 'w1_one=' ~ (multissid_post_summary.dev_sem_ok.w1_one | string) ~ ', '
+                    ~ 'w0_lbd=' ~ (multissid_post_summary.dev_sem_ok.w0_lbd | string) ~ ', '
+                    ~ 'w1_lbd=' ~ (multissid_post_summary.dev_sem_ok.w1_lbd | string) ~ '; '
+                    ~ 'cfg_match=' ~ (multissid_post_summary.cfg_hash_match | string) ~ '; '
+                    ~ 'size(old/new)=' ~ (multissid_post_summary.old_size | string) ~ '/' ~ (multissid_post_summary.new_size | string) ~ '; '
+                    ~ 'nbsp_ssids=' ~ (multissid_post_summary.nbsp_count | string)
+                  )
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+
+    - name: Set NetBox custom field multissidfix=v1.1
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'multissidfix',
+            'task_result': 'v1.1'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    # --- FAILURE PATH (device did not return) ---
+    - name: Journal — restart/SSH probe failed
+      when: not ssh_up
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: restart issued; SSH did not return after 24 x 5s checks — leaving parachute active.'
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Stop host after failed SSH probe
+      when: not ssh_up
+      meta: end_host
+

files/ansible-playbooks/multissidfix.yml

@@ -0,0 +1,802 @@
+---
+- name: "Multi-SSID fix (phase 1: persist + runtime apply)"
+  hosts: all
+  gather_facts: false
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (same contract as wifidebug16.yml)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    # Reboot parachute delay (seconds)
+    parachute_delay_s: 600
+
+    # Local working paths (controller)
+    local_cfg:  "/tmp/{{ inventory_hostname }}_config.json"
+    local_new:  "/tmp/{{ inventory_hostname }}_config.json.new"
+    local_step: "/tmp/{{ inventory_hostname }}_config.step"
+    local_diff: "/tmp/{{ inventory_hostname }}_config.diff"
+    local_dd:   "/tmp/{{ inventory_hostname }}_dd.json"
+
+    # External scripts/filters (next to wifidebug.sh in your repo)
+    dd_script_src: "files/wirelessduediligence.sh"
+    f_1vap0_src:   "files/filter_1vap_id0.jq"
+    f_1vap1_src:   "files/filter_1vap_id1.jq"
+    f_2_01_src:    "files/filter_2vaps_0_1.jq"
+    f_2_10_src:    "files/filter_2vaps_1_0.jq"
+
+    # Controller tmp destinations
+    dd_script: "/tmp/wirelessduediligence.sh"
+    f_1vap0:   "/tmp/filter_1vap_id0.jq"
+    f_1vap1:   "/tmp/filter_1vap_id1.jq"
+    f_2_01:    "/tmp/filter_2vaps_0_1.jq"
+    f_2_10:    "/tmp/filter_2vaps_1_0.jq"
+
+    # Remote (device)
+    remote_cfg:    "/tmp/config.json"
+    remote_new:    "/tmp/config.json.new"
+    remote_backup: "/tmp/config.json.multissidfix1.backup"
+
+  tasks:
+
+    ##########################################################################
+    # a) Connectivity + basic sanity
+    ##########################################################################
+    - name: SSH reachability probe
+      raw: "echo ping"
+      register: ping_result
+      ignore_errors: true
+
+    - name: Stop if SSH unreachable (soft-fail)
+      when: ping_result is failed
+      block:
+        - name: Journal soft-fail (no SSH)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — SSH not reachable'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Ensure remote /tmp/config.json exists
+      raw: "test -s {{ remote_cfg }}"
+      register: remote_cfg_check
+      changed_when: false
+
+    - name: Stop if /tmp/config.json missing (soft-fail)
+      when: remote_cfg_check.rc != 0
+      block:
+        - name: Journal soft-fail (missing config.json)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — missing /tmp/config.json'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    - name: Check controller tools (jq/sshpass/diff/sha256sum)
+      delegate_to: localhost
+      shell: |
+        command -v jq >/dev/null 2>&1 \
+        && command -v sshpass >/dev/null 2>&1 \
+        && command -v diff >/dev/null 2>&1 \
+        && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1)
+      args: { executable: /bin/bash }
+      register: ctrl_tools
+      changed_when: false
+
+    - name: Stop if controller tools missing (soft-fail)
+      when: ctrl_tools.rc != 0
+      block:
+        - name: Journal soft-fail (missing tools)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail — controller missing jq/sshpass/diff/sha256sum'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # b) Due diligence (external script, run on controller)
+    ##########################################################################
+    - name: Copy due diligence script to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ dd_script_src }}"
+        dest: "{{ dd_script }}"
+        mode: "0755"
+
+    - name: Pull /tmp/config.json from device to controller
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_cfg }}
+        {{ local_cfg }}
+
+    - name: Run due diligence on controller
+      delegate_to: localhost
+      shell: "sh {{ dd_script | quote }} {{ local_cfg | quote }} > {{ local_dd | quote }}"
+      args: { executable: /bin/sh }
+      changed_when: false
+
+    - name: Parse due diligence JSON
+      delegate_to: localhost
+      set_fact:
+        dd: "{{ lookup('file', local_dd) | from_json }}"
+
+    # Normalize types for robust `when:` checks
+    - name: Extract & normalize dd facts
+      delegate_to: localhost
+      set_fact:
+        w0_total_i: "{{ (dd.radios.wifi0.vaps_total | int) }}"
+        w1_total_i: "{{ (dd.radios.wifi1.vaps_total | int) }}"
+        w0_idx_i:   "{{ (dd.radios.wifi0.indices | map('int') | list) }}"
+        w1_idx_i:   "{{ (dd.radios.wifi1.indices | map('int') | list) }}"
+
+    ##########################################################################
+    # c) Soft-fail: layout policy checks (one journal)
+    ##########################################################################
+    - name: Init soft-fail reasons (policy)
+      delegate_to: localhost
+      set_fact:
+        softfail_reasons: []
+
+    - name: wifi0 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w0_total_i | int) == 1 and (0 not in w0_idx_i and 1 not in w0_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 ONE_VAP but index not 0/1 (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi0 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w0_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi0 MORE_THAN_TWO_VAPS (indices=' ~ (w0_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if ONE_VAP but index not 0/1
+      delegate_to: localhost
+      when: (w1_total_i | int) == 1 and (0 not in w1_idx_i and 1 not in w1_idx_i)
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 ONE_VAP but index not 0/1 (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if TWO_VAPS but indices not (0,1)/(1,0)
+      delegate_to: localhost
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort not in [[0,1],[1,0]])
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: wifi1 — add reason if MORE_THAN_TWO_VAPS
+      delegate_to: localhost
+      when: (w1_total_i | int) > 2
+      set_fact:
+        softfail_reasons: "{{ softfail_reasons + [ 'wifi1 MORE_THAN_TWO_VAPS (indices=' ~ (w1_idx_i | string) ~ ')' ] }}"
+
+    - name: Publish soft-fail journal & stop host (policy)
+      when: (softfail_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (policy)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (policy) — ' ~ (softfail_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # d) Filters to controller tmp + candidate generation (per radio)
+    ##########################################################################
+    - name: Copy jq filters to controller tmp
+      delegate_to: localhost
+      copy:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        mode: "0644"
+      loop:
+        - { src: "{{ f_1vap0_src }}", dest: "{{ f_1vap0 }}" }
+        - { src: "{{ f_1vap1_src }}", dest: "{{ f_1vap1 }}" }
+        - { src: "{{ f_2_01_src }}",  dest: "{{ f_2_01 }}"  }
+        - { src: "{{ f_2_10_src }}",  dest: "{{ f_2_10 }}"  }
+
+    - name: Start from current config as working file
+      delegate_to: localhost
+      copy:
+        src: "{{ local_cfg }}"
+        dest: "{{ local_step }}"
+        mode: "0644"
+
+    # wifi0 choice
+    - name: wifi0 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (0 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 1 and (1 in w0_idx_i)
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi0 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi0 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w0_total_i | int) == 2 and (w0_idx_i == [1,0])
+      changed_when: true
+
+    # wifi1 choice
+    - name: wifi1 — apply filter 1vap_id0
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (0 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 1vap_id1 (Policy B)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 1 and (1 in w1_idx_i)
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_0_1
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i | sort == [0,1])
+      changed_when: true
+
+    - name: wifi1 — apply filter 2vaps_1_0 (actual order 1,0)
+      delegate_to: localhost
+      shell: "jq --arg r wifi1 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}"
+      args: { executable: /bin/bash }
+      when: (w1_total_i | int) == 2 and (w1_idx_i == [1,0])
+      changed_when: true
+
+    - name: Debug normalized types
+      delegate_to: localhost
+      debug:
+        msg:
+          - "w0_total_i(type)={{ w0_total_i | type_debug }}  value={{ w0_total_i }}"
+          - "w1_total_i(type)={{ w1_total_i | type_debug }}  value={{ w1_total_i }}"
+
+    - name: Move working file to final candidate
+      delegate_to: localhost
+      copy:
+        src: "{{ local_step }}"
+        dest: "{{ local_new }}"
+        mode: "0644"
+
+    ##########################################################################
+    # e) Candidate checks — accumulate reasons; soft-fail once if any
+    ##########################################################################
+    - name: Init soft-fail reasons (candidate)
+      delegate_to: localhost
+      set_fact:
+        cand_reasons: []
+
+    - name: Check JSON validity
+      delegate_to: localhost
+      shell: "jq -e '.' {{ local_new | quote }} >/dev/null"
+      args: { executable: /bin/bash }
+      register: json_valid
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if JSON invalid
+      delegate_to: localhost
+      when: json_valid.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'invalid JSON syntax in candidate' ] }}"
+
+    - name: Semantic — wifi0 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 semantic fails
+      delegate_to: localhost
+      when: sem_w0.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi1 exactly one enabled AP VAP
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 semantic fails
+      delegate_to: localhost
+      when: sem_w1.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 must have exactly one enabled AP VAP' ] }}"
+
+    - name: Semantic — wifi0 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi0.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w0_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi0 lbd check fails
+      delegate_to: localhost
+      when: sem_w0_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi0 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Semantic — wifi1 enabled AP VAP has lbd=true
+      delegate_to: localhost
+      shell: |
+        jq -e '(.wireless.radios.wifi1.vaps // [])
+               | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null
+      args: { executable: /bin/bash }
+      register: sem_w1_lbd
+      changed_when: false
+      failed_when: false
+
+    - name: Add reason if wifi1 lbd check fails
+      delegate_to: localhost
+      when: sem_w1_lbd.rc != 0
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'wifi1 enabled AP VAP must have lbd=true' ] }}"
+
+    - name: Get original file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_cfg }}"
+      register: stat_old
+      changed_when: false
+
+    - name: Get new file size
+      delegate_to: localhost
+      stat:
+        path: "{{ local_new }}"
+      register: stat_new
+      changed_when: false
+
+    - name: Add reason if size delta > 2%
+      delegate_to: localhost
+      when: stat_old.stat.size | int == 0 or
+            ( ((stat_new.stat.size | int) - (stat_old.stat.size | int)) | abs ) > ( (stat_old.stat.size | int) * 0.02 )
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'candidate size delta >2% (old=' ~ (stat_old.stat.size | string) ~ ', new=' ~ (stat_new.stat.size | string) ~ ')' ] }}"
+
+    - name: Build unified diff (first 200 lines)
+      delegate_to: localhost
+      shell: "diff -u {{ local_cfg | quote }} {{ local_new | quote }} | sed -n '1,200p' > {{ local_diff | quote }} || true"
+      args: { executable: /bin/bash }
+      changed_when: false
+
+    - name: Show unified diff (first 200 lines)
+      delegate_to: localhost
+      debug:
+        msg: "{{ lookup('file', local_diff) | default('(no diff output)') }}"
+
+    - name: sha256 (controller) of candidate
+      delegate_to: localhost
+      command: sha256sum {{ local_new }}
+      register: sha_local
+      changed_when: false
+
+    - name: Copy candidate to device temp
+      delegate_to: localhost
+      command: >
+        sshpass -p {{ ssh_pass | quote }}
+        scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+        {{ local_new }}
+        {{ ssh_user }}@{{ ansible_host }}:{{ remote_new }}
+      register: scp_new
+      changed_when: true
+
+    - name: sha256 (remote) of candidate
+      raw: "sha256sum {{ remote_new }} || busybox sha256sum {{ remote_new }}"
+      register: sha_remote
+      changed_when: false
+
+    - name: Add reason if sha256 mismatch
+      delegate_to: localhost
+      when: (sha_local.stdout.split()[0]) != (sha_remote.stdout.split()[0])
+      set_fact:
+        cand_reasons: "{{ cand_reasons + [ 'sha256 mismatch controller vs remote' ] }}"
+
+    - name: Detect SSIDs with NBSP (informational)
+      delegate_to: localhost
+      shell: |
+        nbsp="$(printf '\302\240')"
+        jq -r '..|objects|.ssid? // empty' {{ local_new | quote }} | grep -F "$nbsp" || true
+      args: { executable: /bin/bash }
+      register: nbsp_lines
+      changed_when: false
+
+    - name: Save NBSP report lines
+      set_fact:
+        ssid_nbsp_lines: "{{ nbsp_lines.stdout_lines | default([]) }}"
+
+    - name: Publish soft-fail journal & stop host (candidate issues)
+      when: (cand_reasons | length) > 0
+      block:
+        - name: Journal soft-fail (candidate)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result': 'multissidfix: soft-fail (candidate) — ' ~ (cand_reasons | join('; '))
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+        - meta: end_host
+
+    ##########################################################################
+    # f) Backup, parachute, promote, persist, apply, outcome
+    ##########################################################################
+    - name: Backup current config on device
+      raw: "cp -a {{ remote_cfg }} {{ remote_backup }}"
+      changed_when: true
+
+    # --- PRE-APPLY HEADS-UP ---
+    - name: Journal — candidate validated, will promote/persist/apply shortly
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: candidate passed checks; proceeding to backup + parachute + promote + sysconf -w + system-stop/start. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Arm reboot parachute (BusyBox reboot -d {{ parachute_delay_s }})
+      raw: "sh -c 'trap \"\" HUP; reboot -d {{ parachute_delay_s }} </dev/null >/dev/null 2>&1 &'"
+      changed_when: true
+      ignore_errors: true
+
+    - name: Promote candidate to active config
+      raw: "mv {{ remote_new }} {{ remote_cfg }} && chown root:root {{ remote_cfg }} && chmod 0644 {{ remote_cfg }}"
+      changed_when: true
+
+    # IMPORTANT: persist BEFORE runtime restart
+    - name: Persist config to flash (sysconf -w)
+      raw: "sysconf -w"
+      register: sysconf_write
+      changed_when: true
+      failed_when: false
+
+    # --- PRE-RESTART COMMIT ---
+    - name: Pre-restart journal (promoted & persisted; about to restart from controller)
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: promoted candidate, persisted (sysconf -w rc=' ~ (sysconf_write.rc | default('n/a')) | string ~ '), '
+              ~ 'system-stop/start launching from controller with 10s cap; '
+              ~ 'parachute -d ' ~ parachute_delay_s|string ~ 's armed. '
+              ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string
+              ~ ', nbsp_ssids=' ~ (ssid_nbsp_lines|length)|string
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Launch restart from controller with 10s cap (SSH command)
+      delegate_to: localhost
+      shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+      args:
+        executable: /bin/bash
+      register: restart_kick
+      changed_when: true
+      failed_when: false
+
+    - name: Note restart_kick result (debug)
+      delegate_to: localhost
+      debug:
+        msg:
+          - "restart_kick.rc={{ restart_kick.rc }}"
+          - "stdout(last 5 lines): {{ (restart_kick.stdout_lines | default([]))[-5:] | default([]) }}"
+          - "stderr(last 5 lines): {{ (restart_kick.stderr_lines | default([]))[-5:] | default([]) }}"
+
+    # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+    - name: Probe SSH with nc (24 tries, 5s each)
+      delegate_to: localhost
+      shell: "nc -z -w 3 {{ ansible_host }} 22"
+      register: nc_probe
+      retries: 24
+      delay: 5
+      until: nc_probe.rc == 0
+      changed_when: false
+      failed_when: false
+
+    - name: Set ssh_up fact
+      delegate_to: localhost
+      set_fact:
+        ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+    # --- SUCCESS PATH ---
+    - name: Post-return actions (only if SSH came back)
+      when: ssh_up | bool
+      block:
+
+        - name: Disarm delayed reboot if present
+          raw: "pgrep -x reboot && kill -9 $(pgrep -x reboot) || true"
+          changed_when: true
+          ignore_errors: true
+          register: disarm_reboot
+
+        # Re-run simple semantic checks on the active config
+        - name: Device semantic — wifi0 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 exactly one enabled AP VAP
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi0 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi0.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w0_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: Device semantic — wifi1 enabled AP VAP has lbd=true
+          raw: |
+            jq -e '(.wireless.radios.wifi1.vaps // [])
+                   | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null
+          register: dev_sem_w1_lbd
+          changed_when: false
+          failed_when: false
+
+        - name: sha256 (remote) of current active config (post-return)
+          raw: "sha256sum {{ remote_cfg }} || busybox sha256sum {{ remote_cfg }}"
+          register: sha_remote_after
+          changed_when: false
+          failed_when: false
+
+        - name: Decide post-return checks summary (controller side)
+          delegate_to: localhost
+          set_fact:
+            multissid_post_summary:
+              reboot_disarmed: "{{ (disarm_reboot is defined) and (disarm_reboot.rc is defined) and (disarm_reboot.rc in [0]) }}"
+              persisted_ok: "{{ (sysconf_write.rc | default(0)) == 0 }}"
+              dev_sem_ok:
+                w0_one: "{{ (dev_sem_w0.rc | default(1)) == 0 }}"
+                w1_one: "{{ (dev_sem_w1.rc | default(1)) == 0 }}"
+                w0_lbd: "{{ (dev_sem_w0_lbd.rc | default(1)) == 0 }}"
+                w1_lbd: "{{ (dev_sem_w1_lbd.rc | default(1)) == 0 }}"
+              cfg_hash_match: "{{ (sha_remote_after.stdout.split()[0] | default('')) == (sha_remote.stdout.split()[0] | default('')) }}"
+              old_size: "{{ stat_old.stat.size | default('n/a') }}"
+              new_size: "{{ stat_new.stat.size | default('n/a') }}"
+              nbsp_count: "{{ (ssid_nbsp_lines | default([])) | length }}"
+
+        - name: Publish final success journal (device returned; persisted; checks pass)
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'journal_add',
+                'task_result':
+                  (
+                    'multissidfix: success — device is back; '
+                    ~ (multissid_post_summary.reboot_disarmed | ternary('parachute disarmed; ', 'parachute was not armed; '))
+                    ~ (multissid_post_summary.persisted_ok | ternary('persisted (sysconf -w); ', 'persist failed; '))
+                    ~ 'semantics: '
+                    ~ 'w0_one=' ~ (multissid_post_summary.dev_sem_ok.w0_one | string) ~ ', '
+                    ~ 'w1_one=' ~ (multissid_post_summary.dev_sem_ok.w1_one | string) ~ ', '
+                    ~ 'w0_lbd=' ~ (multissid_post_summary.dev_sem_ok.w0_lbd | string) ~ ', '
+                    ~ 'w1_lbd=' ~ (multissid_post_summary.dev_sem_ok.w1_lbd | string) ~ '; '
+                    ~ 'cfg_match=' ~ (multissid_post_summary.cfg_hash_match | string) ~ '; '
+                    ~ 'size(old/new)=' ~ (multissid_post_summary.old_size | string) ~ '/' ~ (multissid_post_summary.new_size | string) ~ '; '
+                    ~ 'nbsp_ssids=' ~ (multissid_post_summary.nbsp_count | string)
+                  )
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+
+    - name: Set NetBox custom field multissidfix=v1
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'multissidfix',
+            'task_result': 'v1'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    # --- FAILURE PATH (device did not return) ---
+    - name: Journal — restart/SSH probe failed
+      when: not ssh_up
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'multissidfix: restart issued; SSH did not return after 24 x 5s checks — leaving parachute active.'
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Stop host after failed SSH probe
+      when: not ssh_up
+      meta: end_host

files/ansible-playbooks/persuasive-upgrade.yml

@@ -0,0 +1,330 @@
+---
+- name: Persuasive / hunting upgrade orchestrator
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ config
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+
+    # Exchanges / queues
+    work_exchange: "deviceconfig"
+    work_routing_key: "deviceconfig"           # immediate work path
+    holding_exchange: "deviceconfig.holding"
+    holding_routing_key: "persuasive"          # reschedules go to persuasive holding
+    control_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_timeout: 10
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+
+    # Policy defaults (overridable via -e from task_options)
+    pu_period_default: "30m"
+    pu_attempts_default: 48
+    pu_untilhours_default: "72h"
+
+  tasks:
+    - name: Normalize inputs (no self-referential defaults)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        pu_period: "{{ pu_period | default(pu_period_default) }}"
+        pu_attempts: "{{ (pu_attempts | default(pu_attempts_default)) | int }}"
+        pu_untilhours: "{{ pu_untilhours | default(pu_untilhours_default) }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default(lookup('pipe','date -u +%FT%TZ')) }}"
+        controller_now_iso: "{{ lookup('pipe','date -u +%FT%TZ') }}"
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "pu_period={{ pu_period }}"
+          - "pu_attempts={{ pu_attempts }}"
+          - "pu_untilhours={{ pu_untilhours }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "controller_now={{ controller_now_iso }}"
+
+    # Optional time budget
+    - name: Compute budget_ms (supports d/h/m/s in pu_untilhours)
+      ansible.builtin.set_fact:
+        budget_ms: >-
+          {{
+            (
+              (
+                (pu_untilhours | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
+                (pu_untilhours | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
+                (pu_untilhours | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
+                (pu_untilhours | regex_findall('([0-9]+)s') | first | default('0')) | int
+              ) * 1000
+            )
+          }}
+
+    - name: Compute elapsed since original emission (ms)
+      ansible.builtin.set_fact:
+        elapsed_ms: >-
+          {{
+            (
+              (lookup('pipe', 'date -u -d ' ~ controller_now_iso ~ ' +%s') | int) -
+              (lookup('pipe', 'date -u -d ' ~ original_emitted_at ~ ' +%s') | int)
+            ) * 1000
+          }}
+
+    # Helpers for final summary line
+    - name: Compute period_sec from pu_period (supports d/h/m/s)
+      ansible.builtin.set_fact:
+        period_sec: >-
+          {{
+            (
+              (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
+              (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
+              (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
+              (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
+            )
+          }}
+
+    - name: Compute budget_hours string (one decimal)
+      ansible.builtin.set_fact:
+        budget_hours_str: "{{ '%.1f' | format( (budget_ms | int) / 3600000.0 ) }}"
+
+    - name: Gave up due to time budget
+      when: (elapsed_ms | int) >= (budget_ms | int)
+      delegate_to: localhost
+      block:
+        - name: Journal final give-up (unified message)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: >-
+                {{
+                  {
+                    "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                    "task_name": "journal_add",
+                    "task_result": (
+                      "device wasn't reachable for " ~ (attempt | string) ~
+                      " attempts, each " ~ (period_sec | string) ~
+                      " seconds for " ~ budget_hours_str ~
+                      " hours. backing off, won't persuade it more. Schedule again if needed (limit: time budget exhausted)"
+                    )
+                  } | to_json
+                }}
+              payload_encoding: "string"
+          register: rmq_j_budget
+          changed_when: (rmq_j_budget.json is defined) and (rmq_j_budget.json.routed | default(false) | bool)
+
+        - ansible.builtin.meta: end_host
+
+    # Attempt cap
+    - name: Gave up due to attempts cap
+      when: (attempt | int) >= (pu_attempts | int)
+      delegate_to: localhost
+      block:
+        - name: Journal final give-up (unified message)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: >-
+                {{
+                  {
+                    "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                    "task_name": "journal_add",
+                    "task_result": (
+                      "device wasn't reachable for " ~ (attempt | string) ~
+                      " attempts, each " ~ (period_sec | string) ~
+                      " seconds for " ~ budget_hours_str ~
+                      " hours. backing off, won't persuade it more. Schedule again if needed (limit: attempts cap reached)"
+                    )
+                  } | to_json
+                }}
+              payload_encoding: "string"
+          register: rmq_j_gaveup
+          changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+        - ansible.builtin.meta: end_host
+
+    # Probe
+    - name: Check TCP/{{ tcp_port }} via nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # ONLINE → hand off to normal path
+    - name: Publish upgrade-confirmed to deviceconfig (immediate)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ work_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ work_routing_key }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "upgrade-confirmed"
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_start
+      changed_when: (rmq_start.json is defined) and (rmq_start.json.routed | default(false) | bool)
+
+    - name: "Journal: online, starting upgrade"
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": ("persuasive-upgrade: device online, starting upgrade-confirmed now. Original=" ~ original_emitted_at)
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_j_start
+      changed_when: (rmq_j_start.json is defined) and (rmq_j_start.json.routed | default(false) | bool)
+
+    - ansible.builtin.meta: end_host
+      when: nc_probe.rc == 0
+
+    # OFFLINE → reschedule into persuasive holding
+    - name: Compute TTL ms from pu_period (supports d/h/m/s)
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        ttl_ms: >-
+          {{
+            (
+              (
+                (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
+                (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
+                (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
+                (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
+              ) * 1000
+            )
+          }}
+
+    - name: Compute next_attempt
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+
+    - name: Build next task_options (carry policy + increment attempt)
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        next_task_options: >-
+          -e pu_period={{ pu_period }}
+          -e pu_attempts={{ pu_attempts }}
+          -e pu_untilhours={{ pu_untilhours }}
+          -e attempt={{ next_attempt }}
+          -e original_emitted_at='{{ original_emitted_at }}'
+
+    - name: Decide if we should emit the reschedule journal this attempt
+      when: nc_probe.rc != 0
+      ansible.builtin.set_fact:
+        _pu_journal_this_try: "{{ (attempt | int) in [1, 2] or ((attempt | int) % 10 == 0) }}"
+
+    - name: "Publish delayed persuasive-upgrade to holding (routing: persuasive)"
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers:
+              x-delay: "{{ ttl_ms | int }}"
+          routing_key: "{{ holding_routing_key }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "persuasive-upgrade",
+                "task_options": (next_task_options | trim)
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: "Journal: offline, rescheduled"
+      when: nc_probe.rc != 0 and (_pu_journal_this_try | bool)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": ("persuasive-upgrade: device offline, rescheduling (attempt " ~ attempt ~ "/" ~ pu_attempts ~ ", next in " ~ pu_period ~ "). Original=" ~ original_emitted_at)
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_j_off
+      changed_when: (rmq_j_off.json is defined) and (rmq_j_off.json.routed | default(false) | bool)
+
+    - ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0

files/ansible-playbooks/remove-scroll24.yml

@@ -0,0 +1,158 @@
+---
+- name: Remove scroll24 script and cron references
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    remote_script: "/root/scroll24.sh"
+    remote_crontab: "/etc/crontabs/root"
+    tmp_cron_new: "/tmp/cron.root.new"
+    crontab_backup_dir: "/etc/crontabs"
+
+    # RabbitMQ (same contract you use elsewhere)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    ##########################################################################
+    # 1) Make sure crontab file exists (don’t create noise)
+    ##########################################################################
+    - name: Ensure crontab file exists (with perms)
+      raw: |
+        if [ ! -f {{ remote_crontab }} ]; then
+          touch {{ remote_crontab }};
+        fi
+        chown root:root {{ remote_crontab }};
+        chmod 0644 {{ remote_crontab }};
+      changed_when: false
+
+    ##########################################################################
+    # 2) Check whether any scroll24 entries exist
+    ##########################################################################
+    - name: Detect existing scroll24 lines
+      raw: "grep -F 'scroll24.sh' {{ remote_crontab }} || true"
+      register: cron_subset
+      changed_when: false
+      failed_when: false
+
+    - name: Decide if crontab needs cleanup
+      set_fact:
+        crontab_changed: "{{ (cron_subset.stdout | trim) != '' }}"
+
+    - name: Backup current crontab (timestamped)
+      when: crontab_changed | bool
+      raw: "cp -a {{ remote_crontab }} {{ crontab_backup_dir }}/root.bak.$(date +%Y%m%d%H%M%S)"
+      changed_when: true
+
+    - name: Remove scroll24 lines from crontab (preserve others; tidy EOF)
+      when: crontab_changed | bool
+      raw: "grep -v 'scroll24\\.sh' {{ remote_crontab }} > {{ tmp_cron_new }} && awk 'BEGIN{for(i=1;i<=NR;i++)a[i]=$0} {a[NR]=$0} END{e=NR; while(e>0 && a[e] ~ /^[[:space:]]*$/){e--}; for(i=1;i<=e;i++) print a[i]}' {{ tmp_cron_new }} > {{ tmp_cron_new }}.trim && mv {{ tmp_cron_new }}.trim {{ tmp_cron_new }} && printf '\\n' >> {{ tmp_cron_new }} && mv {{ tmp_cron_new }} {{ remote_crontab }} && chown root:root {{ remote_crontab }} && chmod 0644 {{ remote_crontab }}"
+      changed_when: true
+
+    ##########################################################################
+    # 3) Remove /root/scroll24.sh if present
+    ##########################################################################
+    - name: Check if /root/scroll24.sh exists
+      raw: "[ -f {{ remote_script }} ] && echo PRESENT || echo ABSENT"
+      register: script_check
+      changed_when: false
+
+    - name: Remove /root/scroll24.sh
+      when: (script_check.stdout | trim) == 'PRESENT'
+      raw: "rm -f {{ remote_script }}"
+      register: rm_script
+      changed_when: true
+
+    - name: Flag script_removed
+      set_fact:
+        script_removed: "{{ ((script_check.stdout | trim) == 'PRESENT') }}"
+
+    ##########################################################################
+    # 4) If crontab changed, restart crond (with :51–:59 guard)
+    ##########################################################################
+    - name: Get current seconds
+      when: crontab_changed | bool
+      raw: "date +%S"
+      register: nowsec
+      changed_when: false
+
+    - name: Sleep 10s if seconds 51-59
+      when: crontab_changed | bool and (nowsec.stdout | int >= 51)
+      pause:
+        seconds: 10
+
+    - name: Restart crond via move/move
+      when: crontab_changed | bool
+      raw: "mv /tmp/launchd/services/crond /root/crond && sleep 1 && mv /root/crond /tmp/launchd/services/crond"
+      register: crond_restart
+      changed_when: true
+      failed_when: false
+
+    - name: Verify crond is running
+      when: crontab_changed | bool
+      raw: "pgrep -f '/usr/sbin/crond' || busybox pgrep crond || echo missing"
+      register: crond_pid
+      changed_when: false
+      failed_when: false
+
+    ##########################################################################
+    # 5) Journal + reset custom field
+    ##########################################################################
+    - name: Publish removal journal to control queue
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result':
+              (
+                'scroll24: removal — '
+                ~ (crontab_changed | ternary('crontab cleaned; ', 'no crontab refs; '))
+                ~ (script_removed   | ternary('script deleted; ', 'script not present; '))
+                ~ (crontab_changed | ternary('crond restarted', 'crond unchanged'))
+              )
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Clear NetBox custom field scroll24
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'scroll24',
+            'task_result': 'nomore'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/resetradios.yml

@@ -0,0 +1,33 @@
+---
+- name: Reset radios on a host and show brief status
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Bounce ath0 and ath1 with ifconfig
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        ifconfig ath0 down
+        sleep 2
+        ifconfig ath0 up
+        ifconfig ath1 down
+        sleep 1
+        ifconfig ath1 up
+      register: reset_out
+      changed_when: true
+
+    - name: Wait 2s for interfaces to settle (controller-side)
+      ansible.builtin.pause:
+        seconds: 2
+
+    - name: Grab brief ath0/ath1 status (header + one following line)
+      ansible.builtin.raw: |
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        iwconfig 2>/dev/null | grep -E '^(ath0|ath1)\b' -A 1
+      register: iw_out
+      changed_when: false
+
+    - name: Show ath0/ath1 status
+      ansible.builtin.debug:
+        msg: "{{ iw_out.stdout | trim }}"

files/ansible-playbooks/setntptime.yml

@@ -0,0 +1,39 @@
+---
+- name: Simple script to show current version
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: get the date
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        date
+      register: date_out
+      changed_when: true
+
+
+    - name: show the current date
+      ansible.builtin.debug:
+        msg: "{{ date_out.stdout | trim }}"
+
+    - name: use sntp -S to set system time
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        sntp -S time.ikeja.co.za
+      changed_when: true
+
+    - name: get the date
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        date
+      register: date_out
+      changed_when: true
+
+
+    - name: show the current date
+      ansible.builtin.debug:
+        msg: "{{ date_out.stdout | trim }}"
+

files/ansible-playbooks/system-check-version.yml

@@ -0,0 +1,152 @@
+---
+- name: System firmware version check (banner probe + journal)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ (match your existing defaults)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults (kept identical to your afterupgrade_check.yml)
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+  tasks:
+    # -------- Fast TCP reachability probe (controller-side), unchanged style --------
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    # If TCP failed → emit a single journal line and stop
+    - name: Build TCP-fail journal payload
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tcp_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            system check: TCP {{ tcp_port }} unreachable (nc failed)
+
+    - name: Publish TCP-fail journal
+      when: tcp_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tcp_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tcp_fail_resp
+      changed_when: (rmq_tcp_fail_resp.json is defined) and (rmq_tcp_fail_resp.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP failure
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    # -------- SSH banner probe (controller-side), EXACT command reused --------
+    - name: Probe banner via SSH from controller (classic extraction)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    # SSH error → journal and stop
+    - name: Build SSH-fail journal payload
+      when: banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        ssh_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            system check: SSH error - {{ (banner_probe.stderr | default('') | trim) }}
+
+    - name: Publish SSH-fail journal
+      when: ssh_fail_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssh_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssh_fail_resp
+      changed_when: (rmq_ssh_fail_resp.json is defined) and (rmq_ssh_fail_resp.json.routed | default(false) | bool)
+
+    - name: Stop host after SSH failure
+      when: banner_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    # -------- Success: banner line captured → journal with "system check:" prefix --------
+    - name: Build success journal payload (banner captured)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        sc_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            system check: Banner='{{ (banner_probe.stdout | default('') | trim) }}'
+
+    - name: Publish success journal
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ sc_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_sc_success_resp
+      changed_when: (rmq_sc_success_resp.json is defined) and (rmq_sc_success_resp.json.routed | default(false) | bool)
+

files/ansible-playbooks/system-stop-system-start.yml

@@ -0,0 +1,19 @@
+---
+- name: Simple script restart all processes
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: execute and grab output
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        system-stop ; sleep 1; system-start
+      register: ssss_out
+      changed_when: true
+
+    - name: show the output
+      ansible.builtin.debug:
+        msg: "{{ ssss_out.stdout | trim }}"
+
+

files/ansible-playbooks/update-indoor.yml-bckp1

@@ -0,0 +1,997 @@
+---
+# update-secondline.yml
+- name: Second-line upgrade tunneled noninvasive engine
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Busybox-safe PATH prefix for all remote raw calls on DEV1
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # DEV1 credentials (stable, like rebootin222)
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    # Tunnel target DEV2 behind DEV1
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    # Temp IP we add to DEV1 so it can reach DEV2
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+
+    # DEV2 behind the tunnel
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # SSH options used from controller
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
+
+    # Image to stage on DEV2 noninvasive we do not run update -w here
+    image_filename: "fox200-2.2.1-r6801.bin"
+    image_md5: "56b7211709de617e058b98d4204e2562"
+    # Optional SHA256; leave empty to skip SHA256 checks
+    image_sha256: ""
+    dev2_image_dir: "/tmp"
+    dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
+
+    # Reboot delay in minutes integer string if set we will schedule reboot on DEV2
+    rebootin: ""
+
+    # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  pre_tasks:
+    # ------------------------------- Hostname sanity DEV1 -------------------------------
+    - name: Read DEV1 hostname from env busybox safe
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop if DEV1 connected hostname differs from inventory
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 idempotent treat File exists as OK
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug add ip results
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ----------------------------
+    - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+
+    - name: Set dev2_mac from fdb scan (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Remove current ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP for DEV2 on DEV1
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug static ARP result
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Skip ARP add because MAC not found
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- Local tunnel preparation ----------------------------
+    - name: Pick a free local port for the tunnel controller
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+            for i in $(seq 1 25); do
+                p="$(shuf -i 20000-39999 -n 1)"
+                if command -v ss >/dev/null 2>&1; then
+                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                        echo "$p"; return 0
+                    fi
+                else
+                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                        echo "$p"; return 0
+                    fi
+                fi
+            done
+            return 1
+        }
+        pick
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port
+      ansible.builtin.meta: end_host
+      when: (pick_port.stdout | trim | length) == 0
+
+    - name: Record chosen local port and control directory
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build tunnel control socket path
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Debug chosen tunnel params
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "local_port={{ _local_port }}"
+          - "ctrl_dir={{ _ctrl_dir }}"
+          - "ctrl_sock={{ _ctrl_sock }}"
+
+    - name: issue arping with own IP
+      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
+
+    # ---------------------------- Start SSH local forward via DEV1 ----------------------------
+    - name: Start SSH tunnel via DEV1 background ControlMaster
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+            -M -S "{{ _ctrl_sock }}" \
+            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+            "${USER}@${HOST}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    - name: Small delay for tunnel to settle
+      delegate_to: localhost
+      ansible.builtin.wait_for:
+        timeout: 1
+      changed_when: false
+
+    - name: Verify tunnel master running proper O check with destination
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
+      register: tun_check
+      changed_when: false
+
+    - name: Debug tunnel check
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "tunnel_check.rc={{ tun_check.rc }}"
+          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
+
+    # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
+    - name: Sanity confirm tunnel TCP reachability to DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug reachability result
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "nc.rc={{ nc_probe.rc }}"
+          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
+          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
+
+    - name: Stop if tunnel TCP check failed
+      ansible.builtin.meta: end_host
+      when: nc_probe.rc != 0
+
+    - name: Sanity show presence and permissions of passfiles on controller
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
+      register: dev2_ls
+      changed_when: false
+
+    - name: Debug passfiles listing
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ (dev2_ls.stdout | default('')) | trim }}"
+          - "{{ (dev2_ls.stderr | default('')) | trim }}"
+
+    # ---------------------------- Pick DEV2 password root only ----------------------------
+    - name: Try DEV2 login with basicpass root
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass as working passfile
+      when: dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login with basicpass2 root only if first failed
+      when: dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 as working passfile
+      when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Set dev2_passfile_used to NONE if neither worked
+      when: dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    - name: Debug selected DEV2 passfile or NONE
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_passfile_used={{ dev2_passfile_used }}"
+          - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}"
+          - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}"
+
+    - name: Read DEV2 hostname via tunnel (busybox-safe)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
+      args:
+        executable: /bin/bash
+      register: dev2_host_read
+      changed_when: false
+
+    - name: Normalize hostnames for strict compare
+      ansible.builtin.set_fact:
+        _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+
+    - name: Debug hostname bytes (hex)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "inv={{ _inv_hn | tojson }}"
+          - "dev1={{ _dev1_hn | tojson }}"
+          - "dev2={{ _dev2_hn | tojson }}"
+
+    # ---------------- Hostname equality guard (soft-journal and stop) ----------------
+    - name: Enforce DEV2 hostname equals inventory and DEV1
+      block:
+        - name: Fail if DEV2 hostname differs from inventory/DEV1
+          ansible.builtin.fail:
+            msg: >
+              Hostname mismatch: DEV2='{{ _dev2_hn }}',
+              inventory='{{ _inv_hn }}',
+              DEV1='{{ _dev1_hn }}'
+          when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
+      rescue:
+        - name: Initialize journal array for hostname mismatch
+          ansible.builtin.set_fact:
+            _journal: []
+            _blocked: true
+            _prep_blocked: false
+          delegate_to: localhost
+
+        - name: Append hostname mismatch to journal
+          ansible.builtin.set_fact:
+            _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}"
+          delegate_to: localhost
+
+        - name: Build control queue payload for indoor aborted journal (hostname)
+          ansible.builtin.set_fact:
+            journal_indoor_aborted:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
+          delegate_to: localhost
+
+        - name: Publish indoor aborted journal to control queue (hostname)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_indoor_aborted | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_indoor_aborted_hn_resp
+          changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_indoor_aborted_hn_resp.status != 200) or
+            (rmq_journal_indoor_aborted_hn_resp.json is not defined) or
+            (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop host after hostname mismatch
+          ansible.builtin.meta: end_host
+
+    # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
+    - name: Initialize journal and flags
+      ansible.builtin.set_fact:
+        _journal: []
+        _prep_blocked: false
+        _blocked: false
+
+    - name: Build specific marker filename (no .bin)
+      ansible.builtin.set_fact:
+        _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}"
+
+    - name: Check for any preparation markers on DEV2 (wildcard count)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "ls /tmp/prepared_for* 2>/dev/null | wc -l"
+      args:
+        executable: /bin/bash
+      register: dev2_prep_count
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if any prep markers already present
+      when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
+
+    # ---------------------------- DEV2 version firmux primary check ----------------------------
+    - name: Read DEV2 firmux release
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      args:
+        executable: /bin/bash
+      register: dev2_firmux
+      changed_when: false
+      ignore_errors: true
+
+    - name: Debug firmux content DEV2
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+
+    # ---------------------------- Normalize rebootin early ----------------------------
+    - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}"
+        _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}"
+        _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}"
+        _reboot_minutes: >-
+          {{
+            0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'])
+            else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1)
+          }}
+
+    - name: "Normalize rebootin (phase 2: requested flag)"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}"
+
+    # ---------------------------- Journal: indoor start (after we know we can proceed) ----------------------------
+    - name: Build control queue payload for indoor start journal
+      ansible.builtin.set_fact:
+        journal_indoor_start:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: Dev2 is reachable, starting with update.
+            image={{ image_filename }}, md5={{ image_md5 }},
+            firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }},
+            reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }}
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+
+    - name: Publish indoor start journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_start | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_start_resp
+      changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_start_resp.status != 200) or
+        (rmq_journal_indoor_start_resp.json is not defined) or
+        (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
+      when: journal_indoor_start is defined
+      delegate_to: localhost
+
+    # ---------------------------- Stage image on DEV2 no write ----------------------------
+    - name: Check local presence of image file
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ image_filename }}"
+      register: local_img
+
+    - name: Soft-block if image missing locally
+      when: not local_img.stat.exists
+      ansible.builtin.set_fact:
+        _blocked: true
+        _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
+
+    - name: Compute local md5 of the image
+      when: local_img.stat.exists
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        md5sum "{{ image_filename }}" | awk '{print $1}'
+      register: local_md5
+      changed_when: false
+
+    - name: Verify local md5 matches expected
+      when: local_img.stat.exists
+      delegate_to: localhost
+      ansible.builtin.assert:
+        that:
+          - (local_md5.stdout | trim) == image_md5
+        fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
+        success_msg: "Local md5 matches expected"
+
+    # Optional SHA256 local
+    - name: Compute local sha256 of the image (if provided)
+      when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sha256sum "{{ image_filename }}" | awk '{print $1}'
+      register: local_sha256
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if local sha256 mismatch or unavailable
+      when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
+
+    # fw_printenv health before upload (soft-fail)
+    - name: Read fw_printenv size (line count) on DEV2
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "(fw_printenv 2>/dev/null | wc -l) || echo 0"
+      args:
+        executable: /bin/bash
+      register: dev2_fwenv_wc
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if fw_printenv too small
+      when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
+
+    - name: Check existing image md5 on DEV2 NOFILE if absent
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
+      args:
+        executable: /bin/bash
+      register: dev2_md5_before
+      changed_when: false
+
+    - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" scp \
+          -P "$PORT" \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/"
+      args:
+        executable: /bin/bash
+      register: scp_push
+      changed_when: true
+
+    - name: Compute md5 of image on DEV2 after copy or if exists
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
+      args:
+        executable: /bin/bash
+      register: dev2_md5_after
+      changed_when: false
+
+    - name: Verify DEV2 md5 matches expected (soft-fail journal)
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
+
+    # Optional SHA256 remote
+    - name: Compute sha256 of image on DEV2 (if provided)
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
+      args:
+        executable: /bin/bash
+      register: dev2_sha256_after
+      changed_when: false
+      ignore_errors: true
+
+    - name: Soft-block if DEV2 sha256 mismatch/unavailable
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
+
+    - name: Run noninvasive update check on DEV2 update c
+      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "update -c '{{ dev2_image_path }}' 2>&1 || true"
+      args:
+        executable: /bin/bash
+      register: dev2_update_check
+      changed_when: false
+
+    - name: Debug update c output DEV2
+      when: dev2_update_check is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
+
+    - name: Soft-block if update -c did not say valid
+      when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
+      ansible.builtin.set_fact:
+        _prep_blocked: true
+        _blocked: true
+        _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
+
+    # ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
+    - name: Build control queue payload for indoor aborted journal
+      ansible.builtin.set_fact:
+        journal_indoor_aborted:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
+      when: (_blocked | default(false))
+      delegate_to: localhost
+
+    - name: Publish indoor aborted journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_aborted | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_aborted_resp
+      changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_aborted_resp.status != 200) or
+        (rmq_journal_indoor_aborted_resp.json is not defined) or
+        (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
+      when: journal_indoor_aborted is defined
+      delegate_to: localhost
+
+    # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
+    - name: Upgrade write and flip block
+      when: not (_blocked | default(false))
+      block:
+        - name: Write image on DEV2 (this will take a while)
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=0 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "update -w '{{ dev2_image_path }}'"
+          args:
+            executable: /bin/bash
+          register: dev2_up_write
+          changed_when: true
+          failed_when: dev2_up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition on DEV2
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "fw_printenv active | awk -F= '/^active=/{print \$2}'"
+          args:
+            executable: /bin/bash
+          register: dev2_active_before
+          changed_when: false
+          failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
+
+        - name: Determine new active value for DEV2
+          when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
+          ansible.builtin.set_fact:
+            dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ dev2_new_active }}
+          when: dev2_new_active is defined
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "fw_setenv active {{ dev2_new_active }}"
+          args:
+            executable: /bin/bash
+          register: dev2_setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped on DEV2
+          when: dev2_new_active is defined
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "fw_printenv active | awk -F= '/^active=/{print \$2}'"
+          args:
+            executable: /bin/bash
+          register: dev2_active_after
+          changed_when: false
+          failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
+
+        - name: Create specific prep marker on DEV2 (no generic)
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            set -e
+            PORT="{{ _local_port }}"
+            sshpass -f "{{ dev2_passfile_used }}" ssh \
+              -o AddressFamily=inet \
+              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+              -o ConnectTimeout=10 \
+              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+              "touch '{{ _marker_specific }}'"
+          args:
+            executable: /bin/bash
+          register: dev2_marker_write
+          changed_when: true
+          ignore_errors: true
+
+    # ---------------------------- Reboot scheduling (normalized) ----------------------------
+    - name: Schedule delayed reboot on DEV2 minutes
+      when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        SECS="$(( {{ _reboot_minutes | int }} * 60 ))"
+        CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=10 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}"
+      args:
+        executable: /bin/bash
+      register: dev2_reboot_sched
+      changed_when: true
+      ignore_errors: true
+
+    - name: Note reboot request bad format
+      when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
+      ansible.builtin.debug:
+        msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"
+
+    # ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
+    - name: Build control queue payload for indoor updated journal
+      ansible.builtin.set_fact:
+        _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}"
+      when: not (_blocked | default(false))
+      delegate_to: localhost
+
+    - name: Build indoor updated journal payload text
+      ansible.builtin.set_fact:
+        journal_indoor_updated:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: Dev2 updated, scheduling reboot {{
+              'now' if (_reboot_requested and (_reboot_minutes | int) == 0)
+              else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0)
+              else 'not requested'
+            }}.
+            write_done={{ _write_success }},
+            active_before={{ (dev2_active_before.stdout | default('NA')) | trim }},
+            active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}
+      when: not (_blocked | default(false)) and (_write_success | bool)
+      delegate_to: localhost
+
+    - name: Publish indoor updated journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_updated | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_updated_resp
+      changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_updated_resp.status != 200) or
+        (rmq_journal_indoor_updated_resp.json is not defined) or
+        (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
+      when: journal_indoor_updated is defined
+      delegate_to: localhost
+
+    # ---------------------------- summary ----------------------------
+    - name: Summary show key results
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_passfile_used={{ dev2_passfile_used }}"
+          - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+          - "local_image_present={{ local_img.stat.exists | default(false) }}"
+          - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
+          - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
+          - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
+          - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}"
+          - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
+          - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
+          - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
+          - "reboot_requested={{ _reboot_requested | default(false) }}"
+          - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}"
+          - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}"
+          - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
+          - "prep_blocked={{ _prep_blocked | default(false) }}"
+          - "blocked={{ _blocked | default(false) }}"
+          - "journal={{ (_journal | default([])) | join(' || ') }}"
+
+  post_tasks:
+    - name: Cleanup always
+      block:
+        - ansible.builtin.debug:
+            msg: "entering cleanup block"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close tunnel best effort
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove control dir best effort
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          changed_when: del_ip.rc == 0
+          failed_when: >
+            del_ip.rc != 0
+            and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+            and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
+
+    - name: Debug del ip results
+      ansible.builtin.debug:
+        msg:
+          - "del_ip.rc={{ del_ip.rc | default('') }}"
+          - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
+          - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
+      when: del_ip is defined

files/ansible-playbooks/update-reboot.yml

@@ -0,0 +1,4 @@
+---
+# update-reboot.yml — thin wrapper, no var forwarding. 
+# Delegates entirely to the unified updater.
+- import_playbook: update-rebootin222.yml

files/ansible-playbooks/update-rebootin222.yml

@@ -0,0 +1,831 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.2-r9778.bin
+    firmware_sha256: "15bc6f3492321196bb5220014c4441a4dceab2af5f570c46a23140cd37b65a02"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.2-r9778.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: 300
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost

files/ansible-playbooks/upgrade-confirmed.yml

@@ -0,0 +1,4 @@
+- import_playbook: update-rebootin222.yml
+  vars:
+    rebootin: 0
+

files/ansible-playbooks/uptime.yml

@@ -0,0 +1,13 @@
+---
+- name: Run uptime on a host
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Check uptime
+      ansible.builtin.raw: uptime
+      register: uptime_out
+
+    - name: Show result
+      ansible.builtin.debug:
+        var: uptime_out.stdout

files/ansible-playbooks/wifidebug14.yml

@@ -0,0 +1,384 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (from env with defaults)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('amq.default', true) }}"
+    queue2_name: "{{ lookup('env','QUEUE2') | default('queue2', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crontabs -f -l 8' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crontab", "crond -c /etc/crontabs -f -l 8")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            - name: Trigger full restart (system-stop; system-start) in background
+              when: do_edit
+              raw: |
+                 sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start
+              ignore_errors: true
+              changed_when: true
+
+            - name: Short grace delay before waiting
+              when: do_edit
+              delegate_to: localhost
+              pause:
+                seconds: 3
+
+            - name: Try to observe SSH port stopping (best effort)
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: stopped
+                timeout: 30
+                sleep: 2
+              ignore_errors: true
+
+            - name: Wait for SSH port to be accepting connections
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: started
+                timeout: 300
+                sleep: 2
+
+            - name: Reset Ansible SSH connection
+              when: do_edit
+              meta: reset_connection
+
+            - name: Verify command execution after restart (no python)
+              when: do_edit
+              raw: "echo rebooted_ok"
+              register: post_restart_probe
+              retries: 120
+              delay: 2
+              until: post_restart_probe is succeeded
+              changed_when: false
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        - name: Build journal_add payload
+          set_fact:
+            journal_payload_obj:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: "{{ result_status }}"
+
+        - name: Publish journal_add to queue2
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ queue2_name }}"
+              payload: "{{ journal_payload_obj | to_json }}"
+              payload_encoding: "string"
+          delegate_to: localhost
+
+        - name: Build deploy_wifidebug success payload
+          when: result_status == 'SUCCESS_DEPLOYED'
+          set_fact:
+            success_payload_obj:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "success"
+
+        - name: Publish deploy_wifidebug success to queue2
+          when: result_status == 'SUCCESS_DEPLOYED'
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ queue2_name }}"
+              payload: "{{ success_payload_obj | to_json }}"
+              payload_encoding: "string"
+          delegate_to: localhost
+
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"
+

files/ansible-playbooks/wifidebug15.yml

@@ -0,0 +1,447 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v15"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            - name: Trigger full restart (system-stop; system-start) in background
+              when: do_edit
+              raw: |
+                 sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start
+              ignore_errors: true
+              changed_when: true
+
+            - name: Short grace delay before waiting
+              when: do_edit
+              delegate_to: localhost
+              pause:
+                seconds: 3
+
+            - name: Try to observe SSH port stopping (best effort)
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: stopped
+                timeout: 30
+                sleep: 2
+              ignore_errors: true
+
+            - name: Wait for SSH port to be accepting connections
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: started
+                timeout: 300
+                sleep: 2
+
+            - name: Reset Ansible SSH connection
+              when: do_edit
+              meta: reset_connection
+
+            - name: Verify command execution after restart (no python)
+              when: do_edit
+              raw: "echo rebooted_ok"
+              register: post_restart_probe
+              retries: 120
+              delay: 2
+              until: post_restart_probe is succeeded
+              changed_when: false
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/wifidebug16.yml

@@ -0,0 +1,447 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v16"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            - name: Trigger full restart (system-stop; system-start) in background
+              when: do_edit
+              raw: |
+                 sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start
+              ignore_errors: true
+              changed_when: true
+
+            - name: Short grace delay before waiting
+              when: do_edit
+              delegate_to: localhost
+              pause:
+                seconds: 3
+
+            - name: Try to observe SSH port stopping (best effort)
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: stopped
+                timeout: 30
+                sleep: 2
+              ignore_errors: true
+
+            - name: Wait for SSH port to be accepting connections
+              when: do_edit
+              delegate_to: localhost
+              wait_for:
+                host: "{{ ansible_host }}"
+                port: 22
+                state: started
+                timeout: 300
+                sleep: 2
+
+            - name: Reset Ansible SSH connection
+              when: do_edit
+              meta: reset_connection
+
+            - name: Verify command execution after restart (no python)
+              when: do_edit
+              raw: "echo rebooted_ok"
+              register: post_restart_probe
+              retries: 120
+              delay: 2
+              until: post_restart_probe is succeeded
+              changed_when: false
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/wifidebug17.yml

@@ -0,0 +1,476 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v17"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            # ===== ADDED: pre-download MD5 on device to guard download integrity =====
+            - name: MD5 of remote /tmp/config.json (pre-download)
+              when: do_edit
+              raw: "md5sum /tmp/config.json || busybox md5sum /tmp/config.json"
+              register: md5_remote_src
+              changed_when: false
+            # ===== END ADDED =====
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            # ===== ADDED: verify download integrity by comparing MD5s =====
+            - name: MD5 of local downloaded config.json
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json"
+              register: md5_local_src
+              changed_when: false
+
+            - name: Fail if MD5 mismatch on downloaded config.json
+              when:
+                - do_edit
+                - (md5_local_src.stdout.split()[0]) != (md5_remote_src.stdout.split()[0] if (md5_remote_src.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch on downloaded /tmp/config.json — aborting (download integrity check failed)."
+            # ===== END ADDED =====
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            # ===== ADDED: controller-side JSON sanity check (jq) =====
+            - name: Controller JSON sanity check (jq)
+              when: do_edit
+              delegate_to: localhost
+              shell: jq empty "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+              register: jq_local
+              changed_when: false
+              failed_when: jq_local.rc != 0
+            # ===== END ADDED =====
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
+
+            - name: Trigger full restart (system-stop; system-start) from controller with 10s cap
+              when: do_edit
+              delegate_to: localhost
+              shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+              args:
+                executable: /bin/bash
+              register: restart_kick
+              changed_when: true
+              failed_when: false
+
+
+            # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+            - name: Probe SSH with nc (24 tries, 5s each)
+              when: do_edit
+              delegate_to: localhost
+              shell: "nc -z -w 3 {{ ansible_host }} 22"
+              register: nc_probe
+              retries: 24
+              delay: 5
+              until: nc_probe.rc == 0
+              changed_when: false
+              failed_when: false
+
+            - name: Set ssh_up fact (from nc probe)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+            - name: Fail if SSH did not return after restart
+              when:
+                - do_edit
+                - not ssh_up | bool
+              fail:
+                msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
+
+            # ======== END OF SURGICAL CHANGE ========
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/wifidebug17.yml-old

@@ -0,0 +1,441 @@
+---
+- name: Deploy WiFi debug (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    remote_syslog_ip: "102.38.125.161"
+
+    # Program version (0)
+    wdbg_version: "v17"
+
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+############ step 1
+
+            - name: Check if crontab launch is already present in sysstart.lua
+              raw: |
+                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
+              register: sysstart_cron_check
+              changed_when: false
+              failed_when: false
+
+            - name: Announce presence and skip edits
+              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
+              debug:
+                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
+
+            - name: Fetch sysstart.lua via scp (password auth)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+              register: scp_get
+              retries: 3
+              delay: 2
+              until: scp_get.rc == 0
+
+            - name: Edit the file locally
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              lineinfile:
+                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
+                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
+                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
+
+            - name: Push the file back (write to -2)
+              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                /tmp/{{ inventory_hostname }}_sysstart.lua
+                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
+              register: scp_put
+              retries: 3
+              delay: 2
+              until: scp_put.rc == 0
+
+############ step 2
+
+            - name: Compute MD5 of local wifidebug.sh
+              delegate_to: localhost
+              command: md5sum files/wifidebug.sh
+              register: md5_local_wdbg
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/wifidebug.sh
+              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
+              register: md5_remote_wdbg
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if wifidebug.sh needs upload
+              set_fact:
+                upload_wdbg: >-
+                  {{ (md5_remote_wdbg.rc != 0)
+                     or ((md5_local_wdbg.stdout.split()[0])
+                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
+
+            - name: Upload wifidebug.sh via scp (overwrite if changed)
+              when: upload_wdbg | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/wifidebug.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
+              register: scp_wifidebug
+              retries: 3
+              delay: 2
+              until: scp_wifidebug.rc == 0
+
+            - name: Ensure /root/wifidebug.sh is executable and owned by root
+              raw: |
+                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
+
+############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if cron line already present
+              raw: |
+                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+############ step 4
+
+            - name: Skip config.json edit if remote_syslog already matches
+              raw: |
+                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
+                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
+                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
+                  echo "SKIP"
+                else
+                  echo "EDIT"
+                fi
+              register: remote_syslog_check
+              changed_when: false
+
+            - name: Set do_edit flag from skip check
+              set_fact:
+                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
+
+            - name: Fetch /tmp/config.json to controller
+              when: do_edit
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
+                /tmp/{{ inventory_hostname }}_config.json
+
+            - name: Strict in-place style-preserving edit (services.remote_syslog only)
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                awk '
+                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
+                  {
+                    line = $0
+                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
+                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
+                    if (in_rs) {
+                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
+                        done_enabled=1
+                      }
+                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
+                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
+                        done_server=1
+                      }
+                    }
+                    print line
+                    if (in_rs) {
+                      opens = gsub(/{/, "{", $0)
+                      closes = gsub(/}/, "}", $0)
+                      depth += (opens - closes)
+                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
+                    }
+                  }
+                ' "/tmp/{{ inventory_hostname }}_config.json" \
+                > "/tmp/{{ inventory_hostname }}_config.json.new"
+              args:
+                executable: /bin/bash
+
+            - name: Push edited config.json.new to remote temp
+              when: do_edit
+              delegate_to: localhost
+              shell: |
+                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
+                "/tmp/{{ inventory_hostname }}_config.json.new" \
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
+              args:
+                executable: /bin/bash
+
+            - name: Compute MD5 of local .new
+              when: do_edit
+              delegate_to: localhost
+              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
+              register: md5_local
+              changed_when: false
+
+            - name: Compute MD5 of remote .new (no Python)
+              when: do_edit
+              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
+              register: md5_remote
+              changed_when: false
+
+            - name: Fail if MD5 mismatch
+              when:
+                - do_edit
+                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
+              fail:
+                msg: "MD5 mismatch between controller and remote copy — aborting replace."
+
+            - name: Commit new config.json (overwrite original)
+              when: do_edit
+              raw: |
+                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
+                mv /tmp/config.json.new /tmp/config.json
+
+            - name: Apply config to runtime (sysconf -w)
+              when: do_edit
+              raw: |
+                sysconf -w || /usr/sbin/sysconf -w
+              register: sysconf_result
+              changed_when: true
+              failed_when: sysconf_result.rc not in [0]
+
+            # (d) Pre-restart notice to control queue
+            - name: Build pre-restart journal (explain why)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                prereboot_payload:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: >-
+                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
+
+            - name: Publish pre-restart journal to control queue
+              when: do_edit
+              delegate_to: localhost
+              uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ prereboot_payload | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_preboot
+              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
+
+            # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
+
+            - name: Trigger full restart (system-stop; system-start) from controller with 10s cap
+              when: do_edit
+              delegate_to: localhost
+              shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
+              args:
+                executable: /bin/bash
+              register: restart_kick
+              changed_when: true
+              failed_when: false
+
+
+            # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
+            - name: Probe SSH with nc (24 tries, 5s each)
+              when: do_edit
+              delegate_to: localhost
+              shell: "nc -z -w 3 {{ ansible_host }} 22"
+              register: nc_probe
+              retries: 24
+              delay: 5
+              until: nc_probe.rc == 0
+              changed_when: false
+              failed_when: false
+
+            - name: Set ssh_up fact (from nc probe)
+              when: do_edit
+              delegate_to: localhost
+              set_fact:
+                ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
+
+            - name: Fail if SSH did not return after restart
+              when:
+                - do_edit
+                - not ssh_up | bool
+              fail:
+                msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
+
+            # ======== END OF SURGICAL CHANGE ========
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # (a) Set custom field wifidebug -> v15 on control queue
+
+        - name: Build custom-field payload (wifidebug -> version)
+          delegate_to: localhost
+          set_fact:
+            wdbg_cf_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "deploy_wifidebug"
+              task_result: "{{ wdbg_version }}"
+
+        - name: Publish custom-field update to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wdbg_cf_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
+
+        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
+
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"
+

files/ansible.cfg

@@ -0,0 +1,14 @@
+[defaults]
+inventory = ./inventory/netbox.yml
+remote_user = root
+ask_pass = false
+host_key_checking = False
+timeout = 30
+forks = 50
+collections_paths = /opt/containers/ansible-worker/data/collections:/usr/share/ansible/collections
+
+[inventory]
+enable_plugins = auto, yaml, ini, toml, netbox.netbox.nb_inventory
+
+[ssh_connection]
+ssh_args = -o PubkeyAuthentication=no

files/entrypoint.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+APP_DIR="/opt/containers/ansible-worker"
+APP_ROOT="${APP_DIR}/app"
+DATA_DIR="${APP_DIR}/data"
+SRC_DIR="${APP_DIR}/files"          # <-- mounted from host ./files
+BIN_DIR="${APP_ROOT}/bin"
+
+VENV_DIR="${DATA_DIR}/ansible_venv"
+COLL_BASE="${DATA_DIR}/collections"
+COLL_NETBOX_DIR="${COLL_BASE}/ansible_collections/netbox/netbox"
+
+mkdir -p "$DATA_DIR" "$BIN_DIR"
+
+log() { echo "[entrypoint] $*"; }
+
+copy_nbplay() {
+  local src="${SRC_DIR}/nbplay"
+  local dst="${BIN_DIR}/nbplay"
+  if [[ -f "$src" ]]; then
+    log "Installing nbplay -> ${dst}"
+    cp -f "$src" "$dst"
+    chmod 0755 "$dst"
+  else
+    log "WARN: nbplay not found at ${src} (skipping)"
+  fi
+}
+
+copy_rmq_client() {
+  local src="${SRC_DIR}/rabbit-client.sh"
+  local dst="${BIN_DIR}/rabbit-client.sh"
+  if [[ -f "$src" ]]; then
+    log "Installing rabbit-client.sh -> ${dst}"
+    cp -f "$src" "$dst"
+    chmod 0755 "$dst"
+  else
+    log "WARN: rabbit-client.sh not found at ${src} (skipping)"
+  fi
+}
+
+copy_playbooks() {
+  local src_dir="${SRC_DIR}/ansible-playbooks"
+  if [[ -d "$src_dir" ]]; then
+    log "Syncing playbooks from ${src_dir} -> ${APP_ROOT}"
+    if command -v rsync >/dev/null 2>&1; then
+      rsync -a --delete "${src_dir}/" "${APP_ROOT}/"
+    else
+      # clean only top-level *.yml in /app (playbooks), then copy
+      find "${APP_ROOT}" -maxdepth 1 -type f -name '*.yml' -exec rm -f {} +
+      cp -a "${src_dir}/." "${APP_ROOT}/"
+    fi
+  else
+    log "WARN: ${src_dir} does not exist (skipping playbooks copy)"
+  fi
+}
+
+provision_pwfile() {
+  local src_yaml="${SRC_DIR}/ssh.yml"
+  local src_txt="${SRC_DIR}/ssh.txt"
+  local dst="${DATA_DIR}/ssh.yml"
+  if [[ -f "$dst" ]]; then
+    log "Found existing pwfile at ${dst} (leaving as-is)"
+    return
+  fi
+  if [[ -f "$src_yaml" ]]; then
+    log "Seeding password vars from ${src_yaml} -> ${dst}"
+    install -m 600 "$src_yaml" "$dst"
+    return
+  fi
+  if [[ -f "$src_txt" ]]; then
+    log "Wrapping plain password ${src_txt} -> ${dst}"
+    local pass
+    pass="$(head -n1 "$src_txt" | tr -d '\r\n')"
+    umask 177
+    {
+      printf 'ansible_user: %s\n' "${NBPLAY_DEFAULT_USER:-root}"
+      printf 'ansible_ssh_pass: %s\n' "$pass"
+    } > "$dst"
+    return
+  fi
+  log "WARN: no files/ssh.yml or files/ssh.txt found; skipping pwfile seed"
+}
+
+# ---- Copy from mounted files/ (if present) ----
+if [[ -d "${SRC_DIR}" ]]; then
+  log "Found source directory: ${SRC_DIR}"
+  copy_nbplay
+  copy_rmq_client
+  copy_playbooks
+  provision_pwfile
+else
+  log "WARN: Source directory ${SRC_DIR} not present. Did you mount ./files? (see docker-compose.yml)"
+fi
+
+# ---- Venv bootstrap ----
+if [[ ! -x "${VENV_DIR}/bin/ansible" ]]; then
+  log "Creating venv in ${VENV_DIR} and installing packages..."
+  python3 -m venv "${VENV_DIR}"
+  "${VENV_DIR}/bin/python" -m pip install --upgrade pip setuptools wheel
+  "${VENV_DIR}/bin/pip" install \
+    "ansible-core==2.16.*" \
+    "pytz" \
+    "pynetbox==7.*"
+  log "venv bootstrap complete."
+fi
+
+# ---- Collections (persistent path) ----
+if [[ ! -d "${COLL_NETBOX_DIR}" ]]; then
+  log "Installing netbox.netbox collection into ${COLL_BASE} ..."
+  mkdir -p "${COLL_BASE}"
+  "${VENV_DIR}/bin/ansible-galaxy" collection install netbox.netbox -p "${COLL_BASE}"
+  log "netbox.netbox installed."
+fi
+
+# Put venv first on PATH
+export PATH="${VENV_DIR}/bin:${PATH}"
+
+export TZ=${TZ:-Africa/Johannesburg}
+
+log "Startup complete. Executing: $*"
+exec "$@"

files/entrypoint.sh-backup

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+APP_DIR="/opt/containers/ansible-worker"
+DATA_DIR="${APP_DIR}/data"
+VENV_DIR="${DATA_DIR}/ansible_venv"
+COLL_BASE="${DATA_DIR}/collections"
+COLL_NETBOX_DIR="${COLL_BASE}/ansible_collections/netbox/netbox"
+
+mkdir -p "$DATA_DIR"
+
+# Bootstrap venv on first run
+if [ ! -x "${VENV_DIR}/bin/ansible" ]; then
+  echo "[entrypoint] Creating venv in ${VENV_DIR} and installing packages..."
+  python3 -m venv "${VENV_DIR}"
+  "${VENV_DIR}/bin/python" -m pip install --upgrade pip setuptools wheel
+  "${VENV_DIR}/bin/pip" install \
+    "ansible-core==2.16.*" \
+    "pytz" \
+    "pynetbox==7.*"
+  echo "[entrypoint] venv bootstrap complete."
+fi
+
+# Ensure NetBox collection exists in the PERSISTENT collections path
+if [ ! -d "${COLL_NETBOX_DIR}" ]; then
+  echo "[entrypoint] Installing netbox.netbox collection into ${COLL_BASE} ..."
+  mkdir -p "${COLL_BASE}"
+  "${VENV_DIR}/bin/ansible-galaxy" collection install netbox.netbox -p "${COLL_BASE}"
+  echo "[entrypoint] netbox.netbox installed."
+fi
+
+# Put venv first on PATH
+export PATH="${VENV_DIR}/bin:${PATH}"
+
+exec "$@"

files/files/crond-root

@@ -0,0 +1 @@
+*/1 * * * * /root/wifidebug.sh

files/files/crond-root-scroll24

@@ -0,0 +1 @@
+*/20 * * * * /root/scroll24.sh --allbymyself

files/files/filter_1vap_id0.jq

@@ -0,0 +1,8 @@
+# enable+lbd on index 0 (no touching others)
+( . as $root
+| ( $root.wireless.radios[$r].vaps // [] ) as $v
+| if ($v | length) >= 1 then
+    .wireless.radios[$r].vaps[0].enabled = true
+  | .wireless.radios[$r].vaps[0].lbd     = true
+  else . end )
+

files/files/filter_1vap_id1.jq

@@ -0,0 +1,11 @@
+# enable+lbd on index 1 (Policy B)
+( . as $root
+| ( $root.wireless.radios[$r].vaps // [] ) as $v
+| if ($v | length) >= 2 then
+    .wireless.radios[$r].vaps[1].enabled = true
+  | .wireless.radios[$r].vaps[1].lbd     = true
+  else
+    ( .wireless.radios[$r].vaps[1].enabled = true
+    | .wireless.radios[$r].vaps[1].lbd     = true )
+  end )
+

files/files/filter_2vaps_0_1.jq

@@ -0,0 +1,9 @@
+# keep 0 (enable+lbd), disable 1
+( . as $root
+| ( $root.wireless.radios[$r].vaps // [] ) as $v
+| if ($v | length) >= 2 then
+    .wireless.radios[$r].vaps[0].enabled = true
+  | .wireless.radios[$r].vaps[0].lbd     = true
+  | .wireless.radios[$r].vaps[1].enabled = false
+  else . end )
+

files/files/filter_2vaps_1_0.jq

@@ -0,0 +1,9 @@
+# layout 1,0 -> same outcome: keep 0 (enable+lbd), disable 1
+( . as $root
+| ( $root.wireless.radios[$r].vaps // [] ) as $v
+| if ($v | length) >= 2 then
+    .wireless.radios[$r].vaps[0].enabled = true
+  | .wireless.radios[$r].vaps[0].lbd     = true
+  | .wireless.radios[$r].vaps[1].enabled = false
+  else . end )
+

files/files/scroll24.sh

@@ -0,0 +1,371 @@
+#!/bin/sh
+# wifiscan24.sh — 2.4 GHz channel scout for ath1 (BusyBox-safe)
+# Default: probe 1/6/11 (+ current), print recommendation, restore original.
+#
+# Flags:
+#   --goforit            : scan + apply best, policy prefers 1/6/11, verify
+#   --force              : as above, but pre-reset AP (disable/enable) before applying
+#   --allbymyself        : try like --goforit; if apply fails, auto-bounce AP and retry once
+#   --keepbadchannels    : allow 2–5 or 7–10 if they win (only matters with goforit/force/allbymyself)
+#   --chan <N>           : override — switch directly to channel N (ignores scan & policy)
+#                          works alone (no reset) or with --force (preflight reset)
+#   --deep               : deeper scan (longer dwell); works with apply modes
+#
+# Score: score = 7*OBSSavg + 3*CUavg  (lower is better)
+# Ties:  lower OBSSmax → lower (more negative) noise → lower CUavg
+#
+# Apply policy:
+#   By default apply only to {1,6,11}. To allow middle channels {2..5,7..10} add --keepbadchannels
+sleep 20
+
+IFACE="${IFACE:-ath1}"
+CHANNELS="${CHANNELS:-1 6 11}"
+CSA_BEACONS="${CSA_BEACONS:-10}"              # CSA duration (~1s at default beacon interval)
+WAIT_AFTER_SWITCH="${WAIT_AFTER_SWITCH:-2}"   # settle time after CSA during scan/apply
+SAMPLES="${SAMPLES:-5}"                       # samples per channel
+SLEEP_BETWEEN="${SLEEP_BETWEEN:-1}"           # seconds between samples
+
+# Deep-scan presets (used only with --deep)
+DEEP_WAIT="2"
+DEEP_SAMPLES="10"
+DEEP_SLEEP="1"
+
+GOOD_CH_SET="1 6 11"                          # policy-preferred set
+
+GOFORIT=0
+FORCE=0
+ALLBY=0
+DEEP=0
+KEEPBAD=0
+OVERRIDE_CH=""
+
+# ---- arg parse ----
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --goforit|--go|--apply) GOFORIT=1 ;;
+    --force) FORCE=1 ;;
+    --allbymyself) ALLBY=1 ;;
+    --deep) DEEP=1 ;;
+    --keepbadchannels) KEEPBAD=1 ;;
+    --chan)
+      shift
+      [ -n "$1" ] || { echo "Error: --chan requires a channel number." >&2; exit 1; }
+      OVERRIDE_CH="$1"
+      ;;
+    -h|--help)
+      cat <<EOF
+Usage: $0 [--goforit|--force|--allbymyself] [--keepbadchannels] [--deep] [--chan N]
+
+Apply modes (choose one):
+  --goforit         scan + apply best (policy prefers 1/6/11), verify
+  --force           as above, but pre-reset AP before applying
+  --allbymyself     like --goforit; if apply fails, auto-bounce AP and retry once
+
+Extras:
+  --keepbadchannels allow 2–5/7–10 if they win (only with apply modes)
+  --deep            deeper scan dwell (more stable numbers)
+  --chan N          override: switch immediately to channel N (ignores scan & policy)
+EOF
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+# If multiple apply flags passed, pick strongest precedence: FORCE > ALLBY > GOFORIT
+if [ "$FORCE" -eq 1 ]; then
+  GOFORIT=0
+  ALLBY=0
+elif [ "$ALLBY" -eq 1 ]; then
+  GOFORIT=0
+fi
+
+# Deep dwell
+if [ "$DEEP" -eq 1 ]; then
+  WAIT_AFTER_SWITCH="$DEEP_WAIT"
+  SAMPLES="$DEEP_SAMPLES"
+  SLEEP_BETWEEN="$DEEP_SLEEP"
+fi
+
+# ---- helpers ----
+freq_from_ch() { ch="$1"; [ "$ch" -eq 14 ] && { echo 2484; return; }; echo $((2412 + 5*(ch-1))); }
+ch_from_freq() { f="$1"; [ "$f" -eq 2484 ] && { echo 14; return; }; echo $(( (f-2412)/5 + 1 )); }
+
+valid_ch() {
+  case "$1" in
+    ''|*[!0-9]*) return 1 ;;
+    *) [ "$1" -ge 1 ] && [ "$1" -le 14 ] ;;
+  esac
+}
+
+in_good_set() {
+  case " $GOOD_CH_SET " in
+    *" $1 "*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+get_current_freq() {
+  f="$(hostapd_cli -i "$IFACE" status 2>/dev/null | awk -F= '/^freq=/{print $2; exit}')"
+  [ -n "$f" ] || f="$(wifitool "$IFACE" get_rrmutil 2>/dev/null | awk -F': *' '/^channel/{print $2; exit}')"
+  echo "$f"
+}
+
+measure_once() {
+  # prints: "chan_util obss_util noise"
+  wifitool "$IFACE" get_rrmutil 2>/dev/null | awk -F': *' '
+    /^chan util/   {cu=$2+0}
+    /^obss util/   {ob=$2+0}
+    /^noise floor/ {nf=$2+0}
+    END {printf "%d %d %d\n", cu, ob, nf}
+  '
+}
+
+# CSA wrapper (quiet)
+do_csa() {
+  hostapd_cli -i "$IFACE" chan_switch "$CSA_BEACONS" "$1" ht bandwidth=20 blocktx >/dev/null 2>&1
+}
+
+# Verify helper
+verify_freq() {
+  target="$1"
+  sleep "$WAIT_AFTER_SWITCH"
+  cur="$(get_current_freq)"
+  [ "$cur" = "$target" ]
+}
+
+# Self-bounce helper
+ap_bounce() {
+  hostapd_cli -i "$IFACE" disable >/dev/null 2>&1
+  sleep 2
+  hostapd_cli -i "$IFACE" enable  >/dev/null 2>&1
+  sleep 2
+}
+
+# ---- prereqs ----
+command -v hostapd_cli >/dev/null 2>&1 || { echo "hostapd_cli not found"; exit 1; }
+command -v wifitool    >/dev/null 2>&1 || { echo "wifitool not found"; exit 1; }
+
+orig_freq="$(get_current_freq)"; orig_ch="$(ch_from_freq "$orig_freq")"
+
+# ---------- OVERRIDE PATH: --chan N ----------
+if [ -n "$OVERRIDE_CH" ]; then
+  valid_ch "$OVERRIDE_CH" || { echo "Invalid channel: $OVERRIDE_CH (expected 1..14)"; exit 1; }
+  target_freq="$(freq_from_ch "$OVERRIDE_CH")"
+  if [ "$target_freq" = "$orig_freq" ]; then
+    echo "Already on channel $OVERRIDE_CH ($target_freq MHz). No switch needed."
+    exit 0
+  fi
+
+  if [ "$FORCE" -eq 1 ]; then
+    echo "Pre-resetting AP before override…"
+    ap_bounce
+  fi
+
+  echo "Applying override: hostapd_cli -i $IFACE chan_switch $CSA_BEACONS $target_freq ht bandwidth=20 blocktx"
+  do_csa "$target_freq"
+  if verify_freq "$target_freq"; then
+    echo "Switched to ch $OVERRIDE_CH ($target_freq MHz)."
+    exit 0
+  else
+    echo "Override failed to take effect. Try: --chan $OVERRIDE_CH --force"
+    exit 1
+  fi
+fi
+
+# ---------- SCAN PATH ----------
+# unique set incl. current
+unique=""; add(){ case " $unique " in *" $1 "*) : ;; *) unique="$unique $1";; esac; }
+for c in $CHANNELS; do add "$c"; done; add "$orig_ch"
+
+echo "Interface: $IFACE"
+echo "Original:  ch $orig_ch ($orig_freq MHz)"
+[ "$DEEP" -eq 1 ] && echo "Mode:      deep scan (WAIT=$WAIT_AFTER_SWITCH s, SAMPLES=$SAMPLES, SLEEP=$SLEEP_BETWEEN s)"
+echo "Testing:  $unique"
+echo
+
+tmp="$(mktemp)"
+RESTORE=1
+cleanup() {
+  if [ "$RESTORE" = 1 ]; then
+    do_csa "$orig_freq"
+  fi
+  rm -f "$tmp"
+}
+trap 'cleanup' EXIT INT TERM
+
+measure_channel() {
+  ch="$1"; freq="$(freq_from_ch "$ch")"
+  curf="$(get_current_freq)"
+  if [ "$curf" != "$freq" ]; then
+    do_csa "$freq"
+    sleep "$WAIT_AFTER_SWITCH"
+  fi
+
+  cu_min=9999; cu_max=0; cu_sum=0
+  ob_min=9999; ob_max=0; ob_sum=0
+  nf_min=9999; nf_max=-9999; nf_sum=0
+
+  i=0
+  while [ "$i" -lt "$SAMPLES" ]; do
+    set -- $(measure_once)   # $1=cu $2=ob $3=nf
+    cu="$1"; ob="$2"; nf="$3"
+
+    [ "$cu" -lt "$cu_min" ] && cu_min="$cu"; [ "$cu" -gt "$cu_max" ] && cu_max="$cu"; cu_sum=$((cu_sum+cu))
+    [ "$ob" -lt "$ob_min" ] && ob_min="$ob"; [ "$ob" -gt "$ob_max" ] && ob_max="$ob"; ob_sum=$((ob_sum+ob))
+    [ "$nf" -lt "$nf_min" ] && nf_min="$nf"; [ "$nf" -gt "$nf_max" ] && nf_max="$nf"; nf_sum=$((nf_sum+nf))
+
+    i=$((i+1))
+    [ "$i" -lt "$SAMPLES" ] && sleep "$SLEEP_BETWEEN"
+  done
+
+  cu_avg=$((cu_sum / SAMPLES)); ob_avg=$((ob_sum / SAMPLES)); nf_avg=$((nf_sum / SAMPLES))
+
+  printf "ch %2d (%4d MHz):  OBSS min/avg/max=%3d/%3d/%3d   CHutil min/avg/max=%3d/%3d/%3d   Noise avg=%4d dBm\n" \
+    "$ch" "$freq" "$ob_min" "$ob_avg" "$ob_max" "$cu_min" "$cu_avg" "$cu_max" "$nf_avg"
+
+  # machine line: ch ob_avg cu_avg ob_max nf_avg
+  echo "__RESULT__ $ch $ob_avg $cu_avg $ob_max $nf_avg"
+}
+
+# Measure all
+for ch in $unique; do
+  out="$(measure_channel "$ch")"
+  printf "%s\n" "$out" | head -n 1
+  printf "%s\n" "$out" | awk '/^__RESULT__/ {print}' >>"$tmp"
+done
+
+# Pick best overall and best among 1/6/11
+best_any_ch=""; best_any_score=999999; best_any_obmax=9999; best_any_nf=0; best_any_cu=9999
+best_good_ch=""; best_good_score=999999; best_good_obmax=9999; best_good_nf=0; best_good_cu=9999
+
+while read -r _ ch ob cu obmax nf; do
+  score=$(( 7*ob + 3*cu ))
+
+  # overall best
+  if [ "$score" -lt "$best_any_score" ]; then
+    best_any_score="$score"; best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu"
+  elif [ "$score" -eq "$best_any_score" ]; then
+    # Tie-breaker without command groups — ash-safe
+    if [ "$obmax" -lt "$best_any_obmax" ]; then
+      best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu"
+    elif [ "$obmax" -eq "$best_any_obmax" ]; then
+      if [ "$nf" -lt "$best_any_nf" ]; then
+        best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu"
+      elif [ "$nf" -eq "$best_any_nf" ] && [ "$cu" -lt "$best_any_cu" ]; then
+        best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu"
+      fi
+    fi
+  fi
+
+  # best among 1/6/11
+  if in_good_set "$ch"; then
+    if [ "$score" -lt "$best_good_score" ]; then
+      best_good_score="$score"; best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu"
+    elif [ "$score" -eq "$best_good_score" ]; then
+      # Tie-breaker without command groups — ash-safe
+      if [ "$obmax" -lt "$best_good_obmax" ]; then
+        best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu"
+      elif [ "$obmax" -eq "$best_good_obmax" ]; then
+        if [ "$nf" -lt "$best_good_nf" ]; then
+          best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu"
+        elif [ "$nf" -eq "$best_good_nf" ] && [ "$cu" -lt "$best_good_cu" ]; then
+          best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu"
+        fi
+      fi
+    fi
+  fi
+done <"$tmp"
+
+best_any_freq="$(freq_from_ch "$best_any_ch")"
+best_good_freq=""; [ -n "$best_good_ch" ] && best_good_freq="$(freq_from_ch "$best_good_ch")"
+
+echo
+echo ">>> Best by score:        ch $best_any_ch ($best_any_freq MHz)  [score=${best_any_score} = 7*OBSSavg + 3*CUavg]"
+
+# Apply target by policy
+apply_ch=""; apply_freq=""; note="policy prefers 1/6/11"
+if [ "$KEEPBAD" -eq 1 ]; then
+  apply_ch="$best_any_ch"; apply_freq="$best_any_freq"; note="--keepbadchannels set (allow 2–5/7–10)"
+else
+  if [ -n "$best_good_ch" ]; then
+    apply_ch="$best_good_ch"; apply_freq="$best_good_freq"
+  else
+    apply_ch="$best_any_ch"; apply_freq="$best_any_freq"; note="no scorable 1/6/11; falling back to overall best"
+  fi
+fi
+
+echo ">>> Policy target (apply): ch $apply_ch ($apply_freq MHz)  [$note]"
+[ "$apply_freq" = "$orig_freq" ] && APPLY_SAME=1 || APPLY_SAME=0
+
+# ---------- APPLY SECTION ----------
+if [ "$GOFORIT" -eq 1 ] || [ "$FORCE" -eq 1 ] || [ "$ALLBY" -eq 1 ]; then
+  if [ "$APPLY_SAME" -eq 1 ]; then
+    echo "Already on the policy target channel. No switch needed."
+    RESTORE=0
+    exit 0
+  fi
+fi
+
+# Common apply attempt
+try_apply_once() {
+  tgt="$1"
+  echo "Applying (no reset): hostapd_cli -i $IFACE chan_switch $CSA_BEACONS $tgt ht bandwidth=20 blocktx"
+  do_csa "$tgt"
+  if verify_freq "$tgt"; then
+    return 0
+  fi
+  return 1
+}
+
+if [ "$FORCE" -eq 1 ]; then
+  echo "Pre-resetting AP (force)…"
+  ap_bounce
+  echo "Applying after reset: hostapd_cli -i $IFACE chan_switch $CSA_BEACONS $apply_freq ht bandwidth=20 blocktx"
+  do_csa "$apply_freq"
+  if verify_freq "$apply_freq"; then
+    echo "Switched to ch $apply_ch ($apply_freq MHz) after AP reset."
+    RESTORE=0
+    exit 0
+  else
+    echo "Unable to switch automatically at this time. Keeping the original channel."
+    exit 1
+  fi
+elif [ "$ALLBY" -eq 1 ]; then
+  # First: normal CSA
+  if try_apply_once "$apply_freq"; then
+    echo "Switched to ch $apply_ch ($apply_freq MHz)."
+    RESTORE=0
+    exit 0
+  fi
+  # Second: bounce then retry once
+  echo "Apply failed; self-bouncing AP and retrying once…"
+  ap_bounce
+  if try_apply_once "$apply_freq"; then
+    echo "Switched to ch $apply_ch ($apply_freq MHz) after self-bounce."
+    RESTORE=0
+    exit 0
+  fi
+  echo "Retry after self-bounce also failed. Keeping the original channel."
+  exit 1
+elif [ "$GOFORIT" -eq 1 ]; then
+  if try_apply_once "$apply_freq"; then
+    echo "Switched to ch $apply_ch ($apply_freq MHz)."
+    RESTORE=0
+    exit 0
+  fi
+  echo "Cannot switch now; the radio needs a brief reset. Re-run with --allbymyself or --force${KEEPBAD:+ --keepbadchannels}."
+  exit 1
+fi
+
+# ---------- SCAN-ONLY ----------
+echo "Scan-only mode."
+echo "To apply (prefer 1/6/11): $0 --goforit"
+echo "To self-heal on failure:  $0 --allbymyself"
+echo "To apply with reset:      $0 --force"
+echo "To allow middle chans:    add --keepbadchannels with one of the above"
+echo "To override to a chan:    $0 --chan N        (optionally add --force)"

files/files/wifidebug.sh

@@ -0,0 +1,421 @@
+#!/bin/sh
+
+#sleep $((RANDOM % 18 + 3))
+
+SCRIPTVERSION=17
+
+delay=$(( ( $(hexdump -n2 -e '/2 "%u"' /dev/urandom) % 18 ) + 3 ))
+#echo "Sleeping for $delay seconds..."
+sleep $delay
+
+# Get hostname
+HOSTNAME=$(cat /proc/sys/kernel/hostname)
+
+FWVER=$(cat /etc/banner | grep rev | awk '{ print $1 }')
+
+# ===========================================================
+# GPS coordinates management (locks, timeout, 0/0 fallback)
+#   - Writes /tmp/gps_coordinates with:
+#       longitude: <lon 6 decimals>
+#       latitude:  <lat 6 decimals>
+#   - Uses /tmp/gps_coordinates.<epoch>.lock (expires 10m)
+#   - Tracks retries in /tmp/gps_timeout (stop after 3)
+#   - On any retrieval attempt (success/fail) -> exit cycle
+# ===========================================================
+
+COORD_FILE="/tmp/gps_coordinates"
+TIMEOUT_FILE="/tmp/gps_timeout"
+LOCK_MAX_AGE_MIN=10          # minutes
+RETRIEVE_MAX_WAIT=480        # seconds (8 minutes)
+LOCATION_LON="0.000000"
+LOCATION_LAT="0.000000"
+
+# Helper: parse existing coords file (if any)
+read_coords_file() {
+  if [ -f "$COORD_FILE" ]; then
+    # Expect two lines: "longitude: X" and "latitude: Y"
+    local lon lat
+    lon=$(awk -F': *' '/^longitude:/ {print $2; exit}' "$COORD_FILE")
+    lat=$(awk -F': *' '/^latitude:/  {print $2; exit}' "$COORD_FILE")
+    # Validate numeric and ranges; format to 6 decimals
+    if echo "$lon" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then
+      if echo "$lat" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then
+        if awk -v a="$lat" -v b="$lon" 'BEGIN{exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}'; then
+          LOCATION_LON=$(awk -v x="$lon" 'BEGIN{printf "%.6f", x+0}')
+          LOCATION_LAT=$(awk -v x="$lat" 'BEGIN{printf "%.6f", x+0}')
+          return 0
+        fi
+      fi
+    fi
+  fi
+  return 1
+}
+
+# If coords exist and valid -> proceed; else attempt retrieval logic
+if read_coords_file; then
+  :
+else
+  # Check retry threshold
+  GTO=0
+  if [ -f "$TIMEOUT_FILE" ]; then
+    GTO=$(cat "$TIMEOUT_FILE" 2>/dev/null | awk '/^[0-9]+$/ {print $0; exit}')
+    [ -z "$GTO" ] && GTO=0
+  fi
+
+  if [ "$GTO" -ge 3 ]; then
+    # Give up trying; ensure coords file exists with 0/0 and exit this cycle
+    printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp"
+    mv "${COORD_FILE}.tmp" "$COORD_FILE"
+    exit 0
+  fi
+
+  # Clean up stale locks (>10 minutes)
+  find /tmp -maxdepth 1 -type f -name 'gps_coordinates.*.lock' -mmin +"$LOCK_MAX_AGE_MIN" -exec rm -f {} \; 2>/dev/null
+
+  # If any non-stale lock remains, skip this run
+  for f in /tmp/gps_coordinates.*.lock; do
+    [ -e "$f" ] || continue
+    # Found an active (<=10m) lock; skip this cycle
+    exit 0
+  done
+
+  # Create fresh lock and start retrieval
+  NOWTS=$(date +%s)
+  LOCK_FILE="/tmp/gps_coordinates.${NOWTS}.lock"
+  : > "$LOCK_FILE" 2>/dev/null
+
+  CAND="/tmp/gps_candidate.$$"
+  (
+    gpspipe -r | awk -F, '
+      # -------- THE ONLY CHANGE IS HERE: use deglen+1 for minutes --------
+      function dm2dd(dm, hemi, deglen, deg, min, dd) {
+        if (dm=="" || hemi=="") return ""
+        deglen = (hemi=="N" || hemi=="S") ? 2 : 3
+        deg = substr(dm, 1, deglen) + 0
+        min = substr(dm, deglen+1) + 0
+        dd = deg + (min / 60.0)
+        return (hemi=="S" || hemi=="W") ? -dd : dd
+      }
+      # Wait for first valid RMC (A)
+      /^\$..RMC/ && $3=="A" {
+        lat = dm2dd($4, $5)
+        lon = dm2dd($6, $7)
+        if (lat != "" && lon != "") {
+          printf("%.6f %.6f\n", lon, lat)
+          exit
+        }
+      }
+    ' > "$CAND"
+  ) &
+  GPID=$!
+
+  # Wait up to RETRIEVE_MAX_WAIT seconds for candidate to appear
+  waited=0
+  while [ "$waited" -lt "$RETRIEVE_MAX_WAIT" ]; do
+    if [ -s "$CAND" ]; then
+      break
+    fi
+    sleep 1
+    waited=$((waited+1))
+  done
+
+  success=0
+  if [ -s "$CAND" ]; then
+    set -- $(head -n1 "$CAND" 2>/dev/null)
+    cand_lon="$1"
+    cand_lat="$2"
+    if awk -v a="$cand_lat" -v b="$cand_lon" 'BEGIN{ok=(a!="" && b!=""); if (!ok) exit 1; exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}'
+    then
+      printf "longitude: %.6f\nlatitude: %.6f\n" "$cand_lon" "$cand_lat" > "${COORD_FILE}.tmp"
+      mv "${COORD_FILE}.tmp" "$COORD_FILE"
+      echo 0 > "$TIMEOUT_FILE"
+      success=1
+    fi
+  fi
+
+  # Cleanup background gpspipe if still running
+  kill "$GPID" 2>/dev/null
+  sleep 1
+  kill -9 "$GPID" 2>/dev/null
+
+  # Remove lock and temp
+  rm -f "$LOCK_FILE" "$CAND" 2>/dev/null
+
+  # Exit this cycle after any retrieval attempt (success or fail)
+  if [ "$success" -eq 1 ]; then
+    exit 0
+  else
+    printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp"
+    mv "${COORD_FILE}.tmp" "$COORD_FILE"
+    GTO=$((GTO+1))
+    echo "$GTO" > "$TIMEOUT_FILE"
+    exit 0
+  fi
+fi
+
+# At this point, LOCATION_LON/LOCATION_LAT are set (from file or default 0/0)
+
+# Get OBSS utilization value from wifitool
+ATH0_OBSS_UTIL=$(wifitool ath0 get_rrmutil | awk '/obss util/ { print $4 }')
+ATH1_OBSS_UTIL=$(wifitool ath1 get_rrmutil | awk '/obss util/ { print $4 }')
+ATH0_CH_UTIL=$(cfg80211tool ath0 get_chutil | awk -F ':' '{ print $2 }')
+ATH1_CH_UTIL=$(cfg80211tool ath1 get_chutil | awk -F ':' '{ print $2 }')
+
+# -----------------------------------------------------------
+# Count connected (real) clients on ath0 and ath1
+# -----------------------------------------------------------
+
+get_usercount() {
+    IFACE=$1
+    hostapd_cli -i "$IFACE" all_sta 2>/dev/null | awk '
+        /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/ {mac=$0; next}
+        /flags=/ {
+            auth = (index($0, "AUTH") > 0)
+            assoc = (index($0, "ASSOC") > 0)
+            authorized = (index($0, "AUTHORIZED") > 0)
+        }
+        /rx_packets=/ {rx_packets=$1; sub("rx_packets=", "", rx_packets)}
+        /tx_packets=/ {tx_packets=$1; sub("tx_packets=", "", tx_packets)}
+        /inactive_msec=/ {inactive=$1; sub("inactive_msec=", "", inactive)}
+        /connected_time=/ {
+            connected=$1; sub("connected_time=", "", connected)
+            if (auth && assoc && authorized &&
+                rx_packets+0 > 0 && tx_packets+0 > 0 &&
+                inactive+0 < 5000 && connected+0 > 60) {
+                count++
+            }
+        }
+        END { print count+0 }
+    '
+}
+
+ATH0_USERCOUNT=$(get_usercount ath0)
+ATH1_USERCOUNT=$(get_usercount ath1)
+
+# -----------------------------------------------------------
+# Get IP/MAC for br-wan
+# -----------------------------------------------------------
+
+MACADDRESS=$(ip link show br-wan | awk '/link\/ether/ {print $2}')
+IPADDR_MASK=$(ip -4 addr show br-wan | awk '/inet / {print $2}' | head -n 1)
+IPADDRESS=${IPADDR_MASK%/*}
+IPMASK=${IPADDR_MASK#*/}
+
+# -----------------------------------------------------------
+# Get LLDP neighbor switch
+NEIGH_SW=$(grep ikeja /tmp/run/lldp_server.json | grep -E "\-sc|\-as" -m1 | sed -E 's/.*"system_name": "([^"]+)".*/\1/')
+
+# -----------------------------------------------------------
+# Get default GW
+DEFGW=$(ip route show | grep default | awk '{ print $3 }')
+
+# -----------------------------------------------------------
+
+# Run once per iface and reuse the buffer
+ATH0_RAW="$(apstats -v -i ath0 2>/dev/null)"
+ATH1_RAW="$(apstats -v -i ath1 2>/dev/null)"
+
+# ---- ath0 (read from ATH0_RAW) ----
+ATH0_TXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }')
+ATH0_RXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }')
+ATH0_TXUDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }')
+ATH0_TXMBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }')
+ATH0_TXMDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }')
+ATH0_TXBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }')
+ATH0_TXFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Tx failures" | awk '{ print $NF }')
+ATH0_PCKERR=$(printf '%s\n' "$ATH0_RAW" | grep "Packets Errored" | awk '{ print $NF }')
+ATH0_RETR=$(printf '%s\n' "$ATH0_RAW" | grep "Retries" | awk '{ print $NF }')
+ATH0_BCNSUC=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon success" | awk '{ print $NF }')
+ATH0_BCNFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon failed" | awk '{ print $NF }')
+
+ATH0_TXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+ATH0_RXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+
+# ---- ath1 (read from ATH1_RAW) ----
+ATH1_TXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }')
+ATH1_RXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }')
+ATH1_TXUDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }')
+ATH1_TXMBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }')
+ATH1_TXMDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }')
+ATH1_TXBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }')
+ATH1_TXFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Tx failures" | awk '{ print $NF }')
+ATH1_PCKERR=$(printf '%s\n' "$ATH1_RAW" | grep "Packets Errored" | awk '{ print $NF }')
+ATH1_RETR=$(printf '%s\n' "$ATH1_RAW" | grep "Retries" | awk '{ print $NF }')
+ATH1_BCNSUC=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon success" | awk '{ print $NF }')
+ATH1_BCNFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon failed" | awk '{ print $NF }')
+
+ATH1_TXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+ATH1_RXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+
+# -----------------------------------------------------------
+# channels
+# -----------------------------------------------------------
+
+ATH0_CH_NMBR=$(iw dev ath0 info | grep MHz | awk '{ print $2 }')
+ATH0_CH_WIDTH=$(iw dev ath0 info | grep MHz | awk '{ print $6 }')
+ATH0_CH_CNTR=$(iw dev ath0 info | grep MHz | awk '{ print $(NF-1) }')
+
+ATH1_CH_NMBR=$(iw dev ath1 info | grep MHz | awk '{ print $2 }')
+ATH1_CH_WIDTH=$(iw dev ath1 info | grep MHz | awk '{ print $6 }')
+ATH1_CH_CNTR=$(iw dev ath1 info | grep MHz | awk '{ print $(NF-1) }')
+
+# --- ath1 RSSI buckets (gold/silver/bronze/trash) ---
+ATH1_USGLD=0; ATH1_USSLV=0; ATH1_USBRN=0; ATH1_USTRS=0
+_assigns="$(
+  (
+    exec 2>/dev/null
+    hostapd_cli -i ath1 all_sta \
+    | grep 'AUTHORIZED' -B1 -A12 \
+    | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \
+    | awk '
+        BEGIN { g=0; s=0; b=0; t=0 }
+        /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next }
+        /^signal=/ {
+          sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0
+          if      (sig>=-65 && sig<=-20) g++
+          else if (sig>=-75 && sig<=-66) s++
+          else if (sig>=-85 && sig<=-76) b++
+          else if (sig>=-200 && sig<=-86) t++
+        }
+        END {
+          printf "ATH1_USGLD=%d\nATH1_USSLV=%d\nATH1_USBRN=%d\nATH1_USTRS=%d\n", g+0, s+0, b+0, t+0
+        }
+    '
+  ) || true
+)"
+if printf '%s\n' "$_assigns" | awk -F= '
+  NR==1 && $1=="ATH1_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1}
+  NR==2 && $1=="ATH1_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1}
+  NR==3 && $1=="ATH1_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1}
+  NR==4 && $1=="ATH1_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1}
+  END{ exit !(ok1&&ok2&&ok3&&ok4) }
+'; then
+  eval "$_assigns"
+fi
+case "$ATH1_USGLD" in (''|*[!0-9]*) ATH1_USGLD=0;; esac
+case "$ATH1_USSLV" in (''|*[!0-9]*) ATH1_USSLV=0;; esac
+case "$ATH1_USBRN" in (''|*[!0-9]*) ATH1_USBRN=0;; esac
+case "$ATH1_USTRS" in (''|*[!0-9]*) ATH1_USTRS=0;; esac
+
+# --- ath0 RSSI buckets (gold/silver/bronze/trash) ---
+ATH0_USGLD=0; ATH0_USSLV=0; ATH0_USBRN=0; ATH0_USTRS=0
+_assigns="$(
+  (
+    exec 2>/dev/null
+    hostapd_cli -i ath0 all_sta \
+    | grep 'AUTHORIZED' -B1 -A12 \
+    | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \
+    | awk '
+        BEGIN { g=0; s=0; b=0; t=0 }
+        /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next }
+        /^signal=/ {
+          sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0
+          if      (sig>=-65 && sig<=-20) g++
+          else if (sig>=-75 && sig<=-66) s++
+          else if (sig>=-85 && sig<=-76) b++
+          else if (sig>=-200 && sig<=-86) t++
+        }
+        END {
+          printf "ATH0_USGLD=%d\nATH0_USSLV=%d\nATH0_USBRN=%d\nATH0_USTRS=%d\n", g+0, s+0, b+0, t+0
+        }
+    '
+  ) || true
+)"
+if printf '%s\n' "$_assigns" | awk -F= '
+  NR==1 && $1=="ATH0_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1}
+  NR==2 && $1=="ATH0_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1}
+  NR==3 && $1=="ATH0_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1}
+  NR==4 && $1=="ATH0_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1}
+  END{ exit !(ok1&&ok2&&ok3&&ok4) }
+'; then
+  eval "$_assigns"
+fi
+case "$ATH0_USGLD" in (''|*[!0-9]*) ATH0_USGLD=0;; esac
+case "$ATH0_USSLV" in (''|*[!0-9]*) ATH0_USSLV=0;; esac
+case "$ATH0_USBRN" in (''|*[!0-9]*) ATH0_USBRN=0;; esac
+case "$ATH0_USTRS" in (''|*[!0-9]*) ATH0_USTRS=0;; esac
+
+# -----------------------------------------------------------
+# FCS totals/errored
+# -----------------------------------------------------------
+
+WIFISTATS_ATH0_FCSOK=$(wifistats wifi0 2 | grep fcs_ok | awk '{ print $3 }')
+WIFISTATS_ATH0_FCSERR=$(wifistats wifi0 2 | grep fcs_err | awk '{ print $3 }')
+
+WIFISTATS_ATH1_FCSOK=$(wifistats wifi1 2 | grep fcs_ok | awk '{ print $3 }')
+WIFISTATS_ATH1_FCSERR=$(wifistats wifi1 2 | grep fcs_err | awk '{ print $3 }')
+
+# 10-hex-char uuid for correlation (shared between the two parts)
+UUID=$(hexdump -n5 -e '5/1 "%02x"' /dev/urandom)
+
+# ------------------------
+# Part 1: ath0 + shared (+ GPS)
+# ------------------------
+MSG1="hostname=${HOSTNAME} \
+uuid=${UUID} part=1 \
+scrver=${SCRIPTVERSION} \
+ath0_ch_nmbr=${ATH0_CH_NMBR} \
+ath0_ch_width=${ATH0_CH_WIDTH} \
+ath0_ch_cntr=${ATH0_CH_CNTR} \
+ath0_obss_util=${ATH0_OBSS_UTIL} \
+ath0_chutil=${ATH0_CH_UTIL} \
+ath0_txdp=${ATH0_TXDP} \
+ath0_rxdp=${ATH0_RXDP} \
+ath0_txudp=${ATH0_TXUDP} \
+ath0_txmbdp=${ATH0_TXMBDP} \
+ath0_txmdp=${ATH0_TXMDP} \
+ath0_txbdp=${ATH0_TXBDP} \
+ath0_txflr=${ATH0_TXFLR} \
+ath0_txrate=${ATH0_TXRATE} \
+ath0_rxrate=${ATH0_RXRATE} \
+ath0_pckerr=${ATH0_PCKERR} \
+ath0_retr=${ATH0_RETR} \
+ath0_bcnsuc=${ATH0_BCNSUC} \
+ath0_bcnflr=${ATH0_BCNFLR} \
+ath0_usercount=${ATH0_USERCOUNT} \
+ath0_usgld=${ATH0_USGLD} \
+ath0_usslv=${ATH0_USSLV} \
+ath0_usbrn=${ATH0_USBRN} \
+ath0_ustrs=${ATH0_USTRS} \
+ath0_fcsok=${WIFISTATS_ATH0_FCSOK} \
+ath0_fcserr=${WIFISTATS_ATH0_FCSERR} \
+fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW} \
+location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}"
+
+# ------------------------
+# Part 2: ath1 only
+# ------------------------
+MSG2="hostname=${HOSTNAME} \
+uuid=${UUID} part=2 \
+scrver=${SCRIPTVERSION} \
+ath1_ch_nmbr=${ATH1_CH_NMBR} \
+ath1_ch_width=${ATH1_CH_WIDTH} \
+ath1_ch_cntr=${ATH1_CH_CNTR} \
+ath1_obss_util=${ATH1_OBSS_UTIL} \
+ath1_chutil=${ATH1_CH_UTIL} \
+ath1_txdp=${ATH1_TXDP} \
+ath1_rxdp=${ATH1_RXDP} \
+ath1_txudp=${ATH1_TXUDP} \
+ath1_txmbdp=${ATH1_TXMBDP} \
+ath1_txmdp=${ATH1_TXMDP} \
+ath1_txbdp=${ATH1_TXBDP} \
+ath1_txflr=${ATH1_TXFLR} \
+ath1_txrate=${ATH1_TXRATE} \
+ath1_rxrate=${ATH1_RXRATE} \
+ath1_pckerr=${ATH1_PCKERR} \
+ath1_retr=${ATH1_RETR} \
+ath1_bcnsuc=${ATH1_BCNSUC} \
+ath1_bcnflr=${ATH1_BCNFLR} \
+ath1_usercount=${ATH1_USERCOUNT} \
+ath1_usgld=${ATH1_USGLD} \
+ath1_usslv=${ATH1_USSLV} \
+ath1_usbrn=${ATH1_USBRN} \
+ath1_ustrs=${ATH1_USTRS} \
+ath1_fcsok=${WIFISTATS_ATH1_FCSOK} \
+ath1_fcserr=${WIFISTATS_ATH1_FCSERR} \
+fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW} \
+location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}"
+
+# Send messages via syslog (two separate lines, same tag)
+logger -t "debug|${HOSTNAME}" "$MSG1"
+logger -t "debug|${HOSTNAME}" "$MSG2"

files/files/wifidebug12.sh

@@ -0,0 +1,294 @@
+#!/bin/sh
+
+#sleep $((RANDOM % 18 + 3))
+
+SCRIPTVERSION=12
+
+delay=$(( ( $(hexdump -n2 -e '/2 "%u"' /dev/urandom) % 18 ) + 3 ))
+#echo "Sleeping for $delay seconds..."
+sleep $delay
+
+# Get hostname
+HOSTNAME=$(cat /proc/sys/kernel/hostname)
+
+FWVER=$(cat /etc/banner | grep rev | awk '{ print $1 }')
+
+# Get OBSS utilization value from wifitool
+ATH0_OBSS_UTIL=$(wifitool ath0 get_rrmutil | awk '/obss util/ { print $4 }')
+ATH1_OBSS_UTIL=$(wifitool ath1 get_rrmutil | awk '/obss util/ { print $4 }')
+ATH0_CH_UTIL=$(cfg80211tool ath0 get_chutil | awk -F ':' '{ print $2 }')
+ATH1_CH_UTIL=$(cfg80211tool ath1 get_chutil | awk -F ':' '{ print $2 }')
+
+# -----------------------------------------------------------
+# Count connected (real) clients on ath0 and ath1
+# -----------------------------------------------------------
+
+get_usercount() {
+    IFACE=$1
+    hostapd_cli -i "$IFACE" all_sta 2>/dev/null | awk '
+        /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/ {mac=$0; next}
+        /flags=/ {
+            auth = (index($0, "AUTH") > 0)
+            assoc = (index($0, "ASSOC") > 0)
+            authorized = (index($0, "AUTHORIZED") > 0)
+        }
+        /rx_packets=/ {rx_packets=$1; sub("rx_packets=", "", rx_packets)}
+        /tx_packets=/ {tx_packets=$1; sub("tx_packets=", "", tx_packets)}
+        /inactive_msec=/ {inactive=$1; sub("inactive_msec=", "", inactive)}
+        /connected_time=/ {
+            connected=$1; sub("connected_time=", "", connected)
+            if (auth && assoc && authorized &&
+                rx_packets+0 > 0 && tx_packets+0 > 0 &&
+                inactive+0 < 5000 && connected+0 > 60) {
+                count++
+            }
+        }
+        END { print count+0 }
+    '
+}
+
+ATH0_USERCOUNT=$(get_usercount ath0)
+ATH1_USERCOUNT=$(get_usercount ath1)
+
+# -----------------------------------------------------------
+# Get IP/MAC for br-wan
+# -----------------------------------------------------------
+
+# MAC address
+MACADDRESS=$(ip link show br-wan | awk '/link\/ether/ {print $2}')
+
+# IP address and mask (CIDR)
+IPADDR_MASK=$(ip -4 addr show br-wan | awk '/inet / {print $2}' | head -n 1)
+
+# Split into IP and mask separately (optional)
+IPADDRESS=${IPADDR_MASK%/*}
+IPMASK=${IPADDR_MASK#*/}
+
+# -----------------------------------------------------------
+# Get LLDP neighbor switch
+#
+
+NEIGH_SW=$(grep ikeja /tmp/run/lldp_server.json | grep -E "\-sc|\-as" -m1 | sed -E 's/.*"system_name": "([^"]+)".*/\1/')
+
+# -----------------------------------------------------------
+# Get default GW
+# -----------------------------------------------------------
+
+DEFGW=$(ip route show | grep default | awk '{ print $3 }')
+
+# -----------------------------------------------------------
+
+# Run once per iface and reuse the buffer
+ATH0_RAW="$(apstats -v -i ath0 2>/dev/null)"
+ATH1_RAW="$(apstats -v -i ath1 2>/dev/null)"
+
+# ---- ath0 (read from ATH0_RAW) ----
+ATH0_TXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }')
+ATH0_RXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }')
+ATH0_TXUDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }')
+ATH0_TXMBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }')
+ATH0_TXMDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }')
+ATH0_TXBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }')
+ATH0_TXFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Tx failures" | awk '{ print $NF }')
+ATH0_PCKERR=$(printf '%s\n' "$ATH0_RAW" | grep "Packets Errored" | awk '{ print $NF }')
+ATH0_RETR=$(printf '%s\n' "$ATH0_RAW" | grep "Retries" | awk '{ print $NF }')
+ATH0_BCNSUC=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon success" | awk '{ print $NF }')
+ATH0_BCNFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon failed" | awk '{ print $NF }')
+
+ATH0_TXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+ATH0_RXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+
+
+# ---- ath1 (read from ATH1_RAW) ----
+ATH1_TXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }')
+ATH1_RXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }')
+ATH1_TXUDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }')
+ATH1_TXMBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }')
+ATH1_TXMDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }')
+ATH1_TXBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }')
+ATH1_TXFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Tx failures" | awk '{ print $NF }')
+ATH1_PCKERR=$(printf '%s\n' "$ATH1_RAW" | grep "Packets Errored" | awk '{ print $NF }')
+ATH1_RETR=$(printf '%s\n' "$ATH1_RAW" | grep "Retries" | awk '{ print $NF }')
+ATH1_BCNSUC=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon success" | awk '{ print $NF }')
+ATH1_BCNFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon failed" | awk '{ print $NF }')
+
+ATH1_TXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+ATH1_RXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+
+
+
+# -----------------------------------------------------------
+# channels
+# -----------------------------------------------------------
+
+ATH0_CH_NMBR=$(iw dev ath0 info | grep MHz | awk '{ print $2 }')
+ATH0_CH_WIDTH=$(iw dev ath0 info | grep MHz | awk '{ print $6 }')
+ATH0_CH_CNTR=$(iw dev ath0 info | grep MHz | awk '{ print $(NF-1) }')
+
+ATH1_CH_NMBR=$(iw dev ath1 info | grep MHz | awk '{ print $2 }')
+ATH1_CH_WIDTH=$(iw dev ath1 info | grep MHz | awk '{ print $6 }')
+ATH1_CH_CNTR=$(iw dev ath1 info | grep MHz | awk '{ print $(NF-1) }')
+
+
+# --- ath1 RSSI buckets (gold/silver/bronze/trash) ---
+ATH1_USGLD=0; ATH1_USSLV=0; ATH1_USBRN=0; ATH1_USTRS=0
+
+_assigns="$(
+  (
+    exec 2>/dev/null
+    hostapd_cli -i ath1 all_sta \
+    | grep 'AUTHORIZED' -B1 -A12 \
+    | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \
+    | awk '
+        BEGIN { g=0; s=0; b=0; t=0 }
+        /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next }
+        /^signal=/ {
+          sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0
+          if      (sig>=-65 && sig<=-20) g++
+          else if (sig>=-75 && sig<=-66) s++
+          else if (sig>=-85 && sig<=-76) b++
+          else if (sig>=-200 && sig<=-86) t++
+        }
+        END {
+          printf "ATH1_USGLD=%d\nATH1_USSLV=%d\nATH1_USBRN=%d\nATH1_USTRS=%d\n", g+0, s+0, b+0, t+0
+        }
+    '
+  ) || true
+)"
+
+# apply only if well-formed; otherwise keep zeros
+if printf '%s\n' "$_assigns" | awk -F= '
+  NR==1 && $1=="ATH1_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1}
+  NR==2 && $1=="ATH1_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1}
+  NR==3 && $1=="ATH1_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1}
+  NR==4 && $1=="ATH1_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1}
+  END{ exit !(ok1&&ok2&&ok3&&ok4) }
+'; then
+  eval "$_assigns"
+fi
+
+# final sanitize
+case "$ATH1_USGLD" in (''|*[!0-9]*) ATH1_USGLD=0;; esac
+case "$ATH1_USSLV" in (''|*[!0-9]*) ATH1_USSLV=0;; esac
+case "$ATH1_USBRN" in (''|*[!0-9]*) ATH1_USBRN=0;; esac
+case "$ATH1_USTRS" in (''|*[!0-9]*) ATH1_USTRS=0;; esac
+
+########
+
+# --- ath0 RSSI buckets (gold/silver/bronze/trash) ---
+ATH0_USGLD=0; ATH0_USSLV=0; ATH0_USBRN=0; ATH0_USTRS=0
+
+_assigns="$(
+  (
+    exec 2>/dev/null
+    hostapd_cli -i ath0 all_sta \
+    | grep 'AUTHORIZED' -B1 -A12 \
+    | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \
+    | awk '
+        BEGIN { g=0; s=0; b=0; t=0 }
+        /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next }
+        /^signal=/ {
+          sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0
+          if      (sig>=-65 && sig<=-20) g++
+          else if (sig>=-75 && sig<=-66) s++
+          else if (sig>=-85 && sig<=-76) b++
+          else if (sig>=-200 && sig<=-86) t++
+        }
+        END {
+          printf "ATH0_USGLD=%d\nATH0_USSLV=%d\nATH0_USBRN=%d\nATH0_USTRS=%d\n", g+0, s+0, b+0, t+0
+        }
+    '
+  ) || true
+)"
+# apply only if well-formed; otherwise keep zeros
+if printf '%s\n' "$_assigns" | awk -F= '
+  NR==1 && $1=="ATH0_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1}
+  NR==2 && $1=="ATH0_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1}
+  NR==3 && $1=="ATH0_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1}
+  NR==4 && $1=="ATH0_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1}
+  END{ exit !(ok1&&ok2&&ok3&&ok4) }
+'; then
+  eval "$_assigns"
+fi
+
+# final sanitize
+case "$ATH0_USGLD" in (''|*[!0-9]*) ATH0_USGLD=0;; esac
+case "$ATH0_USSLV" in (''|*[!0-9]*) ATH0_USSLV=0;; esac
+case "$ATH0_USBRN" in (''|*[!0-9]*) ATH0_USBRN=0;; esac
+case "$ATH0_USTRS" in (''|*[!0-9]*) ATH0_USTRS=0;; esac
+
+# -----------------------------------------------------------
+# FCS totals/errored
+# -----------------------------------------------------------
+
+WIFISTATS_ATH0_FCSOK=$(wifistats wifi0 2 | grep fcs_ok | awk '{ print $3 }')
+WIFISTATS_ATH0_FCSERR=$(wifistats wifi0 2 | grep fcs_err | awk '{ print $3 }')
+
+WIFISTATS_ATH1_FCSOK=$(wifistats wifi1 2 | grep fcs_ok | awk '{ print $3 }')
+WIFISTATS_ATH1_FCSERR=$(wifistats wifi1 2 | grep fcs_err | awk '{ print $3 }')
+
+
+
+# -----------------------------------------------------------
+# Build message
+# -----------------------------------------------------------
+
+MSG="hostname=${HOSTNAME} \
+scrver=${SCRIPTVERSION} \
+ath1_ch_nmbr=${ATH1_CH_NMBR} \
+ath1_ch_width=${ATH1_CH_WIDTH} \
+ath1_ch_cntr=${ATH1_CH_CNTR} \
+ath1_obss_util=${ATH1_OBSS_UTIL} \
+ath1_chutil=${ATH1_CH_UTIL} \
+ath1_txdp=${ATH1_TXDP} \
+ath1_rxdp=${ATH1_RXDP} \
+ath1_txudp=${ATH1_TXUDP} \
+ath1_txmbdp=${ATH1_TXMBDP} \
+ath1_txmdp=${ATH1_TXMDP} \
+ath1_txbdp=${ATH1_TXBDP} \
+ath1_txflr=${ATH1_TXFLR} \
+ath1_txrate=${ATH1_TXRATE} \
+ath1_rxrate=${ATH1_RXRATE} \
+ath1_pckerr=${ATH1_PCKERR} \
+ath1_retr=${ATH1_RETR} \
+ath1_bcnsuc=${ATH1_BCNSUC} \
+ath1_bcnflr=${ATH1_BCNFLR} \
+ath1_usercount=${ATH1_USERCOUNT} \
+ath1_usgld=${ATH1_USGLD} \
+ath1_usslv=${ATH1_USSLV} \
+ath1_usbrn=${ATH1_USBRN} \
+ath1_ustrs=${ATH1_USTRS} \
+ath1_fcsok=${WIFISTATS_ATH1_FCSOK} \
+ath1_fcserr=${WIFISTATS_ATH1_FCSERR} \
+ath0_ch_nmbr=${ATH0_CH_NMBR} \
+ath0_ch_width=${ATH0_CH_WIDTH} \
+ath0_ch_cntr=${ATH0_CH_CNTR} \
+ath0_obss_util=${ATH0_OBSS_UTIL} \
+ath0_chutil=${ATH0_CH_UTIL} \
+ath0_txdp=${ATH0_TXDP} \
+ath0_rxdp=${ATH0_RXDP} \
+ath0_txudp=${ATH0_TXUDP} \
+ath0_txmbdp=${ATH0_TXMBDP} \
+ath0_txmdp=${ATH0_TXMDP} \
+ath0_txbdp=${ATH0_TXBDP} \
+ath0_txflr=${ATH0_TXFLR} \
+ath0_txrate=${ATH0_TXRATE} \
+ath0_rxrate=${ATH0_RXRATE} \
+ath0_pckerr=${ATH0_PCKERR} \
+ath0_retr=${ATH0_RETR} \
+ath0_bcnsuc=${ATH0_BCNSUC} \
+ath0_bcnflr=${ATH0_BCNFLR} \
+ath0_usercount=${ATH0_USERCOUNT} \
+ath0_usgld=${ATH0_USGLD} \
+ath0_usslv=${ATH0_USSLV} \
+ath0_usbrn=${ATH0_USBRN} \
+ath0_ustrs=${ATH0_USTRS} \
+ath0_fcsok=${WIFISTATS_ATH0_FCSOK} \
+ath0_fcserr=${WIFISTATS_ATH0_FCSERR} \
+fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW}"
+
+
+
+# Send message via syslog
+logger -t "debug|${HOSTNAME}" "$MSG"
+

files/files/wifidebug16.sh

@@ -0,0 +1,420 @@
+#!/bin/sh
+
+#sleep $((RANDOM % 18 + 3))
+
+SCRIPTVERSION=16
+
+delay=$(( ( $(hexdump -n2 -e '/2 "%u"' /dev/urandom) % 18 ) + 3 ))
+#echo "Sleeping for $delay seconds..."
+sleep $delay
+
+# Get hostname
+HOSTNAME=$(cat /proc/sys/kernel/hostname)
+
+FWVER=$(cat /etc/banner | grep rev | awk '{ print $1 }')
+
+# ===========================================================
+# GPS coordinates management (locks, timeout, 0/0 fallback)
+#   - Writes /tmp/gps_coordinates with:
+#       longitude: <lon 6 decimals>
+#       latitude:  <lat 6 decimals>
+#   - Uses /tmp/gps_coordinates.<epoch>.lock (expires 10m)
+#   - Tracks retries in /tmp/gps_timeout (stop after 3)
+#   - On any retrieval attempt (success/fail) -> exit cycle
+# ===========================================================
+
+COORD_FILE="/tmp/gps_coordinates"
+TIMEOUT_FILE="/tmp/gps_timeout"
+LOCK_MAX_AGE_MIN=10          # minutes
+RETRIEVE_MAX_WAIT=480        # seconds (8 minutes)
+LOCATION_LON="0.000000"
+LOCATION_LAT="0.000000"
+
+# Helper: parse existing coords file (if any)
+read_coords_file() {
+  if [ -f "$COORD_FILE" ]; then
+    # Expect two lines: "longitude: X" and "latitude: Y"
+    local lon lat
+    lon=$(awk -F': *' '/^longitude:/ {print $2; exit}' "$COORD_FILE")
+    lat=$(awk -F': *' '/^latitude:/  {print $2; exit}' "$COORD_FILE")
+    # Validate numeric and ranges; format to 6 decimals
+    if echo "$lon" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then
+      if echo "$lat" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then
+        if awk -v a="$lat" -v b="$lon" 'BEGIN{exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}'; then
+          LOCATION_LON=$(awk -v x="$lon" 'BEGIN{printf "%.6f", x+0}')
+          LOCATION_LAT=$(awk -v x="$lat" 'BEGIN{printf "%.6f", x+0}')
+          return 0
+        fi
+      fi
+    fi
+  fi
+  return 1
+}
+
+# If coords exist and valid -> proceed; else attempt retrieval logic
+if read_coords_file; then
+  :
+else
+  # Check retry threshold
+  GTO=0
+  if [ -f "$TIMEOUT_FILE" ]; then
+    GTO=$(cat "$TIMEOUT_FILE" 2>/dev/null | awk '/^[0-9]+$/ {print $0; exit}')
+    [ -z "$GTO" ] && GTO=0
+  fi
+
+  if [ "$GTO" -ge 3 ]; then
+    # Give up trying; ensure coords file exists with 0/0 and exit this cycle
+    printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp"
+    mv "${COORD_FILE}.tmp" "$COORD_FILE"
+    exit 0
+  fi
+
+  # Clean up stale locks (>10 minutes)
+  find /tmp -maxdepth 1 -type f -name 'gps_coordinates.*.lock' -mmin +"$LOCK_MAX_AGE_MIN" -exec rm -f {} \; 2>/dev/null
+
+  # If any non-stale lock remains, skip this run
+  for f in /tmp/gps_coordinates.*.lock; do
+    [ -e "$f" ] || continue
+    # Found an active (<=10m) lock; skip this cycle
+    exit 0
+  done
+
+  # Create fresh lock and start retrieval
+  NOWTS=$(date +%s)
+  LOCK_FILE="/tmp/gps_coordinates.${NOWTS}.lock"
+  : > "$LOCK_FILE" 2>/dev/null
+
+  CAND="/tmp/gps_candidate.$$"
+  (
+    gpspipe -r | awk -F, '
+      # -------- THE ONLY CHANGE IS HERE: use deglen+1 for minutes --------
+      function dm2dd(dm, hemi, deglen, deg, min, dd) {
+        if (dm=="" || hemi=="") return ""
+        deglen = (hemi=="N" || hemi=="S") ? 2 : 3
+        deg = substr(dm, 1, deglen) + 0
+        min = substr(dm, deglen+1) + 0
+        dd = deg + (min / 60.0)
+        return (hemi=="S" || hemi=="W") ? -dd : dd
+      }
+      # Wait for first valid RMC (A)
+      /^\$..RMC/ && $3=="A" {
+        lat = dm2dd($4, $5)
+        lon = dm2dd($6, $7)
+        if (lat != "" && lon != "") {
+          printf("%.6f %.6f\n", lon, lat)
+          exit
+        }
+      }
+    ' > "$CAND"
+  ) &
+  GPID=$!
+
+  # Wait up to RETRIEVE_MAX_WAIT seconds for candidate to appear
+  waited=0
+  while [ "$waited" -lt "$RETRIEVE_MAX_WAIT" ]; do
+    if [ -s "$CAND" ]; then
+      break
+    fi
+    sleep 1
+    waited=$((waited+1))
+  done
+
+  success=0
+  if [ -s "$CAND" ]; then
+    set -- $(head -n1 "$CAND" 2>/dev/null)
+    cand_lon="$1"
+    cand_lat="$2"
+    if awk -v a="$cand_lat" -v b="$cand_lon" 'BEGIN{ok=(a!="" && b!=""); if (!ok) exit 1; exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}'
+    then
+      printf "longitude: %.6f\nlatitude: %.6f\n" "$cand_lon" "$cand_lat" > "${COORD_FILE}.tmp"
+      mv "${COORD_FILE}.tmp" "$COORD_FILE"
+      echo 0 > "$TIMEOUT_FILE"
+      success=1
+    fi
+  fi
+
+  # Cleanup background gpspipe if still running
+  kill "$GPID" 2>/dev/null
+  sleep 1
+  kill -9 "$GPID" 2>/dev/null
+
+  # Remove lock and temp
+  rm -f "$LOCK_FILE" "$CAND" 2>/dev/null
+
+  # Exit this cycle after any retrieval attempt (success or fail)
+  if [ "$success" -eq 1 ]; then
+    exit 0
+  else
+    printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp"
+    mv "${COORD_FILE}.tmp" "$COORD_FILE"
+    GTO=$((GTO+1))
+    echo "$GTO" > "$TIMEOUT_FILE"
+    exit 0
+  fi
+fi
+
+# At this point, LOCATION_LON/LOCATION_LAT are set (from file or default 0/0)
+
+# Get OBSS utilization value from wifitool
+ATH0_OBSS_UTIL=$(wifitool ath0 get_rrmutil | awk '/obss util/ { print $4 }')
+ATH1_OBSS_UTIL=$(wifitool ath1 get_rrmutil | awk '/obss util/ { print $4 }')
+ATH0_CH_UTIL=$(cfg80211tool ath0 get_chutil | awk -F ':' '{ print $2 }')
+ATH1_CH_UTIL=$(cfg80211tool ath1 get_chutil | awk -F ':' '{ print $2 }')
+
+# -----------------------------------------------------------
+# Count connected (real) clients on ath0 and ath1
+# -----------------------------------------------------------
+
+get_usercount() {
+    IFACE=$1
+    hostapd_cli -i "$IFACE" all_sta 2>/dev/null | awk '
+        /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/ {mac=$0; next}
+        /flags=/ {
+            auth = (index($0, "AUTH") > 0)
+            assoc = (index($0, "ASSOC") > 0)
+            authorized = (index($0, "AUTHORIZED") > 0)
+        }
+        /rx_packets=/ {rx_packets=$1; sub("rx_packets=", "", rx_packets)}
+        /tx_packets=/ {tx_packets=$1; sub("tx_packets=", "", tx_packets)}
+        /inactive_msec=/ {inactive=$1; sub("inactive_msec=", "", inactive)}
+        /connected_time=/ {
+            connected=$1; sub("connected_time=", "", connected)
+            if (auth && assoc && authorized &&
+                rx_packets+0 > 0 && tx_packets+0 > 0 &&
+                inactive+0 < 5000 && connected+0 > 60) {
+                count++
+            }
+        }
+        END { print count+0 }
+    '
+}
+
+ATH0_USERCOUNT=$(get_usercount ath0)
+ATH1_USERCOUNT=$(get_usercount ath1)
+
+# -----------------------------------------------------------
+# Get IP/MAC for br-wan
+# -----------------------------------------------------------
+
+MACADDRESS=$(ip link show br-wan | awk '/link\/ether/ {print $2}')
+IPADDR_MASK=$(ip -4 addr show br-wan | awk '/inet / {print $2}' | head -n 1)
+IPADDRESS=${IPADDR_MASK%/*}
+IPMASK=${IPADDR_MASK#*/}
+
+# -----------------------------------------------------------
+# Get LLDP neighbor switch
+NEIGH_SW=$(grep ikeja /tmp/run/lldp_server.json | grep -E "\-sc|\-as" -m1 | sed -E 's/.*"system_name": "([^"]+)".*/\1/')
+
+# -----------------------------------------------------------
+# Get default GW
+DEFGW=$(ip route show | grep default | awk '{ print $3 }')
+
+# -----------------------------------------------------------
+
+# Run once per iface and reuse the buffer
+ATH0_RAW="$(apstats -v -i ath0 2>/dev/null)"
+ATH1_RAW="$(apstats -v -i ath1 2>/dev/null)"
+
+# ---- ath0 (read from ATH0_RAW) ----
+ATH0_TXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }')
+ATH0_RXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }')
+ATH0_TXUDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }')
+ATH0_TXMBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }')
+ATH0_TXMDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }')
+ATH0_TXBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }')
+ATH0_TXFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Tx failures" | awk '{ print $NF }')
+ATH0_PCKERR=$(printf '%s\n' "$ATH0_RAW" | grep "Packets Errored" | awk '{ print $NF }')
+ATH0_RETR=$(printf '%s\n' "$ATH0_RAW" | grep "Retries" | awk '{ print $NF }')
+ATH0_BCNSUC=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon success" | awk '{ print $NF }')
+ATH0_BCNFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon failed" | awk '{ print $NF }')
+
+ATH0_TXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+ATH0_RXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+
+# ---- ath1 (read from ATH1_RAW) ----
+ATH1_TXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }')
+ATH1_RXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }')
+ATH1_TXUDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }')
+ATH1_TXMBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }')
+ATH1_TXMDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }')
+ATH1_TXBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }')
+ATH1_TXFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Tx failures" | awk '{ print $NF }')
+ATH1_PCKERR=$(printf '%s\n' "$ATH1_RAW" | grep "Packets Errored" | awk '{ print $NF }')
+ATH1_RETR=$(printf '%s\n' "$ATH1_RAW" | grep "Retries" | awk '{ print $NF }')
+ATH1_BCNSUC=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon success" | awk '{ print $NF }')
+ATH1_BCNFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon failed" | awk '{ print $NF }')
+
+ATH1_TXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+ATH1_RXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}')
+
+# -----------------------------------------------------------
+# channels
+# -----------------------------------------------------------
+
+ATH0_CH_NMBR=$(iw dev ath0 info | grep MHz | awk '{ print $2 }')
+ATH0_CH_WIDTH=$(iw dev ath0 info | grep MHz | awk '{ print $6 }')
+ATH0_CH_CNTR=$(iw dev ath0 info | grep MHz | awk '{ print $(NF-1) }')
+
+ATH1_CH_NMBR=$(iw dev ath1 info | grep MHz | awk '{ print $2 }')
+ATH1_CH_WIDTH=$(iw dev ath1 info | grep MHz | awk '{ print $6 }')
+ATH1_CH_CNTR=$(iw dev ath1 info | grep MHz | awk '{ print $(NF-1) }')
+
+# --- ath1 RSSI buckets (gold/silver/bronze/trash) ---
+ATH1_USGLD=0; ATH1_USSLV=0; ATH1_USBRN=0; ATH1_USTRS=0
+_assigns="$(
+  (
+    exec 2>/dev/null
+    hostapd_cli -i ath1 all_sta \
+    | grep 'AUTHORIZED' -B1 -A12 \
+    | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \
+    | awk '
+        BEGIN { g=0; s=0; b=0; t=0 }
+        /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next }
+        /^signal=/ {
+          sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0
+          if      (sig>=-65 && sig<=-20) g++
+          else if (sig>=-75 && sig<=-66) s++
+          else if (sig>=-85 && sig<=-76) b++
+          else if (sig>=-200 && sig<=-86) t++
+        }
+        END {
+          printf "ATH1_USGLD=%d\nATH1_USSLV=%d\nATH1_USBRN=%d\nATH1_USTRS=%d\n", g+0, s+0, b+0, t+0
+        }
+    '
+  ) || true
+)"
+if printf '%s\n' "$_assigns" | awk -F= '
+  NR==1 && $1=="ATH1_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1}
+  NR==2 && $1=="ATH1_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1}
+  NR==3 && $1=="ATH1_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1}
+  NR==4 && $1=="ATH1_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1}
+  END{ exit !(ok1&&ok2&&ok3&&ok4) }
+'; then
+  eval "$_assigns"
+fi
+case "$ATH1_USGLD" in (''|*[!0-9]*) ATH1_USGLD=0;; esac
+case "$ATH1_USSLV" in (''|*[!0-9]*) ATH1_USSLV=0;; esac
+case "$ATH1_USBRN" in (''|*[!0-9]*) ATH1_USBRN=0;; esac
+case "$ATH1_USTRS" in (''|*[!0-9]*) ATH1_USTRS=0;; esac
+
+# --- ath0 RSSI buckets (gold/silver/bronze/trash) ---
+ATH0_USGLD=0; ATH0_USSLV=0; ATH0_USBRN=0; ATH0_USTRS=0
+_assigns="$(
+  (
+    exec 2>/dev/null
+    hostapd_cli -i ath0 all_sta \
+    | grep 'AUTHORIZED' -B1 -A12 \
+    | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \
+    | awk '
+        BEGIN { g=0; s=0; b=0; t=0 }
+        /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next }
+        /^signal=/ {
+          sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0
+          if      (sig>=-65 && sig<=-20) g++
+          else if (sig>=-75 && sig<=-66) s++
+          else if (sig>=-85 && sig<=-76) b++
+          else if (sig>=-200 && sig<=-86) t++
+        }
+        END {
+          printf "ATH0_USGLD=%d\nATH0_USSLV=%d\nATH0_USBRN=%d\nATH0_USTRS=%d\n", g+0, s+0, b+0, t+0
+        }
+    '
+  ) || true
+)"
+if printf '%s\n' "$_assigns" | awk -F= '
+  NR==1 && $1=="ATH0_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1}
+  NR==2 && $1=="ATH0_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1}
+  NR==3 && $1=="ATH0_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1}
+  NR==4 && $1=="ATH0_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1}
+  END{ exit !(ok1&&ok2&&ok3&&ok4) }
+'; then
+  eval "$_assigns"
+fi
+case "$ATH0_USGLD" in (''|*[!0-9]*) ATH0_USGLD=0;; esac
+case "$ATH0_USSLV" in (''|*[!0-9]*) ATH0_USSLV=0;; esac
+case "$ATH0_USBRN" in (''|*[!0-9]*) ATH0_USBRN=0;; esac
+case "$ATH0_USTRS" in (''|*[!0-9]*) ATH0_USTRS=0;; esac
+
+# -----------------------------------------------------------
+# FCS totals/errored
+# -----------------------------------------------------------
+
+WIFISTATS_ATH0_FCSOK=$(wifistats wifi0 2 | grep fcs_ok | awk '{ print $3 }')
+WIFISTATS_ATH0_FCSERR=$(wifistats wifi0 2 | grep fcs_err | awk '{ print $3 }')
+
+WIFISTATS_ATH1_FCSOK=$(wifistats wifi1 2 | grep fcs_ok | awk '{ print $3 }')
+WIFISTATS_ATH1_FCSERR=$(wifistats wifi1 2 | grep fcs_err | awk '{ print $3 }')
+
+# 10-hex-char uuid for correlation (shared between the two parts)
+UUID=$(hexdump -n5 -e '5/1 "%02x"' /dev/urandom)
+
+# ------------------------
+# Part 1: ath0 + shared (+ GPS)
+# ------------------------
+MSG1="hostname=${HOSTNAME} \
+uuid=${UUID} part=1 \
+scrver=${SCRIPTVERSION} \
+ath0_ch_nmbr=${ATH0_CH_NMBR} \
+ath0_ch_width=${ATH0_CH_WIDTH} \
+ath0_ch_cntr=${ATH0_CH_CNTR} \
+ath0_obss_util=${ATH0_OBSS_UTIL} \
+ath0_chutil=${ATH0_CH_UTIL} \
+ath0_txdp=${ATH0_TXDP} \
+ath0_rxdp=${ATH0_RXDP} \
+ath0_txudp=${ATH0_TXUDP} \
+ath0_txmbdp=${ATH0_TXMBDP} \
+ath0_txmdp=${ATH0_TXMDP} \
+ath0_txbdp=${ATH0_TXBDP} \
+ath0_txflr=${ATH0_TXFLR} \
+ath0_txrate=${ATH0_TXRATE} \
+ath0_rxrate=${ATH0_RXRATE} \
+ath0_pckerr=${ATH0_PCKERR} \
+ath0_retr=${ATH0_RETR} \
+ath0_bcnsuc=${ATH0_BCNSUC} \
+ath0_bcnflr=${ATH0_BCNFLR} \
+ath0_usercount=${ATH0_USERCOUNT} \
+ath0_usgld=${ATH0_USGLD} \
+ath0_usslv=${ATH0_USSLV} \
+ath0_usbrn=${ATH0_USBRN} \
+ath0_ustrs=${ATH0_USTRS} \
+ath0_fcsok=${WIFISTATS_ATH0_FCSOK} \
+ath0_fcserr=${WIFISTATS_ATH0_FCSERR} \
+fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW} \
+location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}"
+
+# ------------------------
+# Part 2: ath1 only
+# ------------------------
+MSG2="hostname=${HOSTNAME} \
+uuid=${UUID} part=2 \
+scrver=${SCRIPTVERSION} \
+ath1_ch_nmbr=${ATH1_CH_NMBR} \
+ath1_ch_width=${ATH1_CH_WIDTH} \
+ath1_ch_cntr=${ATH1_CH_CNTR} \
+ath1_obss_util=${ATH1_OBSS_UTIL} \
+ath1_chutil=${ATH1_CH_UTIL} \
+ath1_txdp=${ATH1_TXDP} \
+ath1_rxdp=${ATH1_RXDP} \
+ath1_txudp=${ATH1_TXUDP} \
+ath1_txmbdp=${ATH1_TXMBDP} \
+ath1_txmdp=${ATH1_TXMDP} \
+ath1_txbdp=${ATH1_TXBDP} \
+ath1_txflr=${ATH1_TXFLR} \
+ath1_txrate=${ATH1_TXRATE} \
+ath1_rxrate=${ATH1_RXRATE} \
+ath1_pckerr=${ATH1_PCKERR} \
+ath1_retr=${ATH1_RETR} \
+ath1_bcnsuc=${ATH1_BCNSUC} \
+ath1_bcnflr=${ATH1_BCNFLR} \
+ath1_usercount=${ATH1_USERCOUNT} \
+ath1_usgld=${ATH1_USGLD} \
+ath1_usslv=${ATH1_USSLV} \
+ath1_usbrn=${ATH1_USBRN} \
+ath1_ustrs=${ATH1_USTRS} \
+ath1_fcsok=${WIFISTATS_ATH1_FCSOK} \
+ath1_fcserr=${WIFISTATS_ATH1_FCSERR} \
+location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}"
+
+# Send messages via syslog (two separate lines, same tag)
+logger -t "debug|${HOSTNAME}" "$MSG1"
+logger -t "debug|${HOSTNAME}" "$MSG2"

files/files/wirelessduediligence.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+# Usage: ./duediligence.sh [/path/to/config.json]
+set -eu
+
+CFG="${1:-/tmp/config.json}"
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo '{"error":"jq not found in PATH"}'
+  exit 2
+fi
+
+if [ ! -f "$CFG" ]; then
+  # Still emit JSON so Ansible can parse the error
+  printf '{"error":"config not found: %s"}\n' "$CFG"
+  exit 2
+fi
+
+jq -c '
+  def classify(n):
+    if n==0 then "NO_VAPS"
+    elif n==1 then "ONE_VAP"
+    elif n==2 then "TWO_VAPS"
+    else "MORE_THAN_TWO_VAPS" end;
+
+  def radio_summary($r):
+    ( .wireless.radios[$r] // {} ) as $rad
+    | ( $rad.vaps // [] ) as $v
+    | {
+        radio_enabled: ($rad.enabled // false),
+        radio_mode:    ($rad.mode // "unknown"),
+        vaps_total:    ($v | length),
+        indices:       ( $v | to_entries | map(.key) ),
+        enabled_ap_indices:
+          ( $v
+            | to_entries
+            | map(select(.value.mode=="ap" and (.value.enabled==true)) | .key)
+          ),
+        classification: classify($v | length),
+        vaps:
+          ( $v
+            | to_entries
+            | map({
+                index: .key,
+                mode: (.value.mode // "null"),
+                enabled: (.value.enabled // false),
+                lbd: (.value.lbd // null),
+                zone: (.value.network.zone // null),
+                ssid: (.value.ssid // "")
+              })
+          )
+      };
+
+  {
+    file: input_filename,
+    radios: {
+      wifi0: radio_summary("wifi0"),
+      wifi1: radio_summary("wifi1"),
+      wlan0: radio_summary("wlan0")
+    },
+    global: {
+      preferred5G: (.wireless.lbd.preferred5G // null)
+    }
+  }
+' "$CFG"

files/group_vars/all.yml

@@ -0,0 +1,5 @@
+ansible_user: root        # or the user your devices accept
+ansible_port: 22
+host_key_checking: false  # optional: you already have this in ansible.cfg
+# ansible_ssh_private_key_file: ~/.ssh/id_rsa   # if you use a key
+

files/inventory/netbox.yml

@@ -0,0 +1,15 @@
+# inventory/netbox.yml
+plugin: netbox.netbox.nb_inventory
+
+api_endpoint: "http://netbox.gt-tiso.ikeja.co.za"   # <-- no trailing /api here
+token: "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+
+query_filters:
+  - status: active
+  - has_primary_ip: True
+
+compose:
+  ansible_host: >-
+    {{ (primary_ip4.address | default('')) | regex_replace('/\\d+$', '') }}
+
+

files/nbplay

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Hardcoded NetBox access (DEV ONLY!)
+NETBOX_URL="http://netbox.gt-tiso.ikeja.co.za"
+NETBOX_TOKEN="7648e4f5ee370cda7834682e61b47c2ee8e95623"
+
+usage() {
+  cat >&2 <<EOF
+Usage: nbplay [--pwfile PATH] [--user USER] [--rebootin VALUE] <playbook.yml> <device_name> [ansible-playbook args...]
+  --pwfile PATH   Path to password/vars file:
+                    - YAML/JSON: must contain ansible_ssh_pass (and optionally ansible_user)
+                    - Plain text: first line is the password (wrapped as ansible_ssh_pass)
+  --user USER     ansible_user to use (overrides file value unless omitted)
+  --rebootin VAL  Passes playbook var rebootin=VAL (e.g. "23:35" or "120")
+Examples:
+  nbplay update-indoor.yml ikeja12345 --pwfile /opt/.../ssh.yml --rebootin "23:35"
+  nbplay update-indoor.yml ikeja12345 --pwfile /opt/.../ssh.txt --user root --rebootin "120"
+EOF
+  exit 2
+}
+
+# ---- Parse args
+PWFILE=""
+USER_OVERRIDE=""
+REBOOTIN_VALUE=""
+ARGS=()
+while (( $# )); do
+  case "${1:-}" in
+    --pwfile) shift; PWFILE="${1:-}"; [[ -n "$PWFILE" ]] || usage; shift;;
+    --user)   shift; USER_OVERRIDE="${1:-}"; [[ -n "$USER_OVERRIDE" ]] || usage; shift;;
+    --rebootin) shift; REBOOTIN_VALUE="${1:-}"; [[ -n "$REBOOTIN_VALUE" ]] || usage; shift;;
+    -h|--help) usage;;
+    *) ARGS+=("$1"); shift;;
+  esac
+done
+set -- "${ARGS[@]}"
+
+if [[ $# -lt 2 ]]; then usage; fi
+PLAYBOOK="$1"; shift
+NAME="$1"; shift
+
+if [[ -z "${PWFILE}" ]]; then
+  DEFAULT_PWFILE="${NBPLAY_PWFILE:-/opt/containers/ansible-worker/data/ssh.yml}"
+  if [[ -f "$DEFAULT_PWFILE" ]]; then
+    PWFILE="$DEFAULT_PWFILE"
+  fi
+fi
+
+# ---- Deps
+command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 127; }
+command -v curl >/dev/null 2>&1 || { echo "curl not found" >&2; exit 127; }
+
+BASE="${NETBOX_URL%/}"
+hdr=(-H "Authorization: Token ${NETBOX_TOKEN}" -H "Accept: application/json")
+
+# ---- Resolve device -> IP
+dev_json="$(curl -fsS "${hdr[@]}" "${BASE}/api/dcim/devices/?name=${NAME}&limit=1")"
+count="$(printf '%s' "$dev_json" | jq -r '.count // 0')"
+if [[ "$count" != "1" ]]; then
+  echo "Device '${NAME}' not found or not unique (count=${count})." >&2
+  exit 1
+fi
+addr_v4="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip4.address // empty')"
+addr_v6="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip6.address // empty')"
+addr_any="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip.address // empty')"
+addr="${addr_v4:-${addr_v6:-${addr_any:-}}}"
+[[ -n "$addr" && "$addr" != "null" ]] || { echo "Device '${NAME}' has no primary IP." >&2; exit 1; }
+ip="${addr%%/*}"
+
+# ---- Temp inventory
+tmpinv="$(mktemp)"; trap 'rm -f "$tmpinv" "$tmpvars"' EXIT
+cat >"$tmpinv" <<EOF
+[nb]
+$NAME ansible_host=$ip
+EOF
+
+# ---- Build extra-vars without leaking secrets in ps
+extravars=()
+tmpvars=""
+
+# Helper: ensure tmpvars exists and is 600
+ensure_tmpvars() {
+  if [[ -z "$tmpvars" ]]; then
+    tmpvars="$(mktemp)"
+    chmod 600 "$tmpvars"
+    extravars+=(-e "@${tmpvars}")
+  fi
+}
+
+if [[ -n "$PWFILE" ]]; then
+  if [[ "$PWFILE" =~ \.(ya?ml|json)$ ]]; then
+    extravars+=(-e "@${PWFILE}")
+  else
+    [[ -r "$PWFILE" ]] || { echo "Password file not readable: $PWFILE" >&2; exit 1; }
+    pass="$(head -n1 "$PWFILE" | tr -d '\r\n')"
+    ensure_tmpvars
+    {
+      echo "ansible_ssh_pass: \"$pass\""
+      [[ -n "$USER_OVERRIDE" ]] && echo "ansible_user: \"$USER_OVERRIDE\""
+    } >> "$tmpvars"
+  fi
+fi
+
+# If user override provided and not already set via tmpvars/YAML, add it safely
+if [[ -n "$USER_OVERRIDE" && -z "$tmpvars" ]]; then
+  ensure_tmpvars
+  echo "ansible_user: \"$USER_OVERRIDE\"" >> "$tmpvars"
+fi
+
+# If rebootin provided, add it to the temp vars file
+if [[ -n "$REBOOTIN_VALUE" ]]; then
+  ensure_tmpvars
+  echo "rebootin: \"$REBOOTIN_VALUE\"" >> "$tmpvars"
+fi
+
+# ---- Run playbook (do NOT use --ask-pass)
+ANSIBLE_ASK_PASS=False exec ansible-playbook -i "$tmpinv" "$PLAYBOOK" -l "$NAME" \
+  "${extravars[@]}" "$@"

files/nbplay_beforeIndoors

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Hardcoded NetBox access (DEV ONLY!)
+NETBOX_URL="http://netbox.gt-tiso.ikeja.co.za"
+NETBOX_TOKEN="7648e4f5ee370cda7834682e61b47c2ee8e95623"
+
+usage() {
+  cat >&2 <<EOF
+Usage: nbplay [--pwfile PATH] [--user USER] <playbook.yml> <device_name> [ansible-playbook args...]
+  --pwfile PATH   Path to password/vars file:
+                    - YAML/JSON: must contain ansible_ssh_pass (and optionally ansible_user)
+                    - Plain text: first line is the password (wrapped as ansible_ssh_pass)
+  --user USER     ansible_user to use (overrides file value unless omitted)
+Examples:
+  nbplay uptime.yml ikeja12345 --pwfile /opt/containers/ansible-worker/data/ssh.yml
+  nbplay uptime.yml ikeja12345 --pwfile /opt/.../ssh.txt --user root
+EOF
+  exit 2
+}
+
+# ---- Parse args
+PWFILE=""
+USER_OVERRIDE=""
+ARGS=()
+while (( $# )); do
+  case "${1:-}" in
+    --pwfile) shift; PWFILE="${1:-}"; [[ -n "$PWFILE" ]] || usage; shift;;
+    --user)   shift; USER_OVERRIDE="${1:-}"; [[ -n "$USER_OVERRIDE" ]] || usage; shift;;
+    -h|--help) usage;;
+    *) ARGS+=("$1"); shift;;
+  esac
+done
+set -- "${ARGS[@]}"
+
+if [[ $# -lt 2 ]]; then usage; fi
+PLAYBOOK="$1"; shift
+NAME="$1"; shift
+
+if [[ -z "${PWFILE}" ]]; then
+  DEFAULT_PWFILE="${NBPLAY_PWFILE:-/opt/containers/ansible-worker/data/ssh.yml}"
+  if [[ -f "$DEFAULT_PWFILE" ]]; then
+    PWFILE="$DEFAULT_PWFILE"
+  fi
+fi
+
+# ---- Deps
+command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 127; }
+command -v curl >/dev/null 2>&1 || { echo "curl not found" >&2; exit 127; }
+
+BASE="${NETBOX_URL%/}"
+hdr=(-H "Authorization: Token ${NETBOX_TOKEN}" -H "Accept: application/json")
+
+# ---- Resolve device -> IP
+dev_json="$(curl -fsS "${hdr[@]}" "${BASE}/api/dcim/devices/?name=${NAME}&limit=1")"
+count="$(printf '%s' "$dev_json" | jq -r '.count // 0')"
+if [[ "$count" != "1" ]]; then
+  echo "Device '${NAME}' not found or not unique (count=${count})." >&2
+  exit 1
+fi
+addr_v4="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip4.address // empty')"
+addr_v6="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip6.address // empty')"
+addr_any="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip.address // empty')"
+addr="${addr_v4:-${addr_v6:-${addr_any:-}}}"
+[[ -n "$addr" && "$addr" != "null" ]] || { echo "Device '${NAME}' has no primary IP." >&2; exit 1; }
+ip="${addr%%/*}"
+
+# ---- Temp inventory
+tmpinv="$(mktemp)"; trap 'rm -f "$tmpinv" "$tmpvars"' EXIT
+cat >"$tmpinv" <<EOF
+[nb]
+$NAME ansible_host=$ip
+EOF
+
+# ---- Build extra-vars without leaking secrets in ps
+extravars=()
+tmpvars=""
+
+if [[ -n "$PWFILE" ]]; then
+  if [[ "$PWFILE" =~ \.(ya?ml|json)$ ]]; then
+    # Must contain: ansible_ssh_pass (and optionally ansible_user)
+    extravars+=(-e "@${PWFILE}")
+  else
+    # Wrap plain text password into a tiny YAML
+    [[ -r "$PWFILE" ]] || { echo "Password file not readable: $PWFILE" >&2; exit 1; }
+    pass="$(head -n1 "$PWFILE" | tr -d '\r\n')"
+    tmpvars="$(mktemp)"
+    {
+      echo "ansible_ssh_pass: \"$pass\""
+      [[ -n "$USER_OVERRIDE" ]] && echo "ansible_user: \"$USER_OVERRIDE\""
+    } > "$tmpvars"
+    chmod 600 "$tmpvars"
+    extravars+=(-e "@${tmpvars}")
+  fi
+fi
+
+# If user override provided and not already set via tmpvars/YAML, add it safely
+if [[ -n "$USER_OVERRIDE" && -z "$tmpvars" ]]; then
+  tmpvars="$(mktemp)"
+  echo "ansible_user: \"$USER_OVERRIDE\"" > "$tmpvars"
+  chmod 600 "$tmpvars"
+  extravars+=(-e "@${tmpvars}")
+fi
+
+# ---- Run playbook (do NOT use --ask-pass)
+ANSIBLE_ASK_PASS=False exec ansible-playbook -i "$tmpinv" "$PLAYBOOK" -l "$NAME" \
+  "${extravars[@]}" "$@"

files/rabbit-client-old.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env or flags) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Mode A: read from a queue (default)
+QUEUE="${QUEUE:-queue1}"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"        # e.g. "amq.topic" or "my-exchange". Leave empty to skip binding mode.
+ROUTING_KEY="${ROUTING_KEY:-#}" # pattern for topic/direct, default is catch-all "#"
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Pretty-print with jq if available
+USE_JQ="${USE_JQ:-auto}"        # auto|yes|no
+USE_COLOR="${USE_COLOR:-yes}"   # yes|no -> 'yes' forces color with jq -C
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+pretty() {
+  if [[ "$USE_JQ" == "yes" ]] || { [[ "$USE_JQ" == "auto" ]] && has_jq; }; then
+    if [[ "$USE_COLOR" == "yes" ]]; then jq -C .; else jq .; fi
+  else
+    cat
+  fi
+}
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log() { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err() { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Map task_name -> absolute playbook path
+map_playbook() {
+  case "$1" in
+    uptime)      printf '%s/uptime.yml' "${APP_ROOT}" ;;
+    reset_aths)  printf '%s/resetradios.yml' "${APP_ROOT}" ;;
+    *)           return 1 ;;
+  esac
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task delay playbook
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  delay="$(jq -r  '.task_delay // empty' <<<"$json")" || delay=""
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  if ! playbook="$(map_playbook "$task")"; then
+    warn "unknown task_name='$task' for device='$device' (no-op)."
+    return 0
+  fi
+
+  # Optional delay: "none" -> no delay; integer -> sleep seconds
+  if [[ -n "$delay" && "$delay" != "none" ]]; then
+    if [[ "$delay" =~ ^[0-9]+$ ]]; then
+      log "delaying ${delay}s before running '${task}' on '${device}'"
+      sleep "$delay"
+    else
+      warn "task_delay value '$delay' not numeric/'none' (ignoring)."
+    fi
+  fi
+
+  log "→ Running playbook $(basename "$playbook") on host '${device}'"
+  if ! "${NBPLAY}" "$playbook" "$device"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # do not exit; keep consuming
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide queue source ---
+if [[ -n "$EXCHANGE" ]]; then
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+else
+  log "Consuming directly from queue: $QUEUE (vhost: $VHOST)"
+fi
+
+log "Press Ctrl+C to stop."
+
+# --- Consume loop ---
+while :; do
+  RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+    "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+  }')"
+
+  # Empty array => no messages
+  if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+    sleep "$SLEEP_SECS"
+    continue
+  fi
+
+  # Process one-by-one (the API returns an array)
+  if has_jq && { [[ "$USE_JQ" == "yes" ]] || [[ "$USE_JQ" == "auto" ]]; }; then
+    echo "$RESP" | jq -c '.[]' | while read -r item; do
+      payload=$(printf '%s' "$item" | jq -r '.payload')
+      rk=$(printf '%s' "$item" | jq -r '.routing_key')
+      ex=$(printf '%s' "$item" | jq -r '.exchange')
+
+      echo "-----"
+      echo "exchange: ${ex:-\"\"}"
+      echo "routing_key: $rk"
+      echo "payload:"
+
+      # Decode + pretty print + DISPATCH
+      if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+        ptype="$(printf '%s' "$payload" | jq -r 'type')"
+        if [[ "$ptype" == "string" ]]; then
+          decoded="$(printf '%s' "$payload" | jq -r .)"
+          if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+            printf '%s' "$decoded" | pretty
+            dispatch_task "$decoded"
+          else
+            printf '%s\n' "$decoded"
+            # not JSON -> no dispatch
+          fi
+        else
+          printf '%s' "$payload" | pretty
+          dispatch_task "$payload"
+        fi
+      else
+        printf '%s\n' "$payload"
+        # not JSON -> no dispatch
+      fi
+    done
+  else
+    # Minimal fallback: print raw response only
+    echo "$RESP"
+  fi
+done

files/rabbit-client.sh

@@ -0,0 +1,287 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Hard override as in your current file
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default). Supports CSV for multiple queues.
+# CHANGED: default now includes both normal and persuasive work queues.
+QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}"    # e.g. "queue_deviceconfig,queue_persuasive"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log()  { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
+  if [[ "$task" == "afterupgrade_check" ]]; then
+    local attempt corr_id orig_at target_ver
+    attempt="$(jq -r '.attempt // empty' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
+    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+
+    [[ -n "$attempt"  ]] && task_options+=" -e attempt=${attempt}"
+    [[ -n "$corr_id"  ]] && task_options+=" -e correlation_id=${corr_id}"
+    [[ -n "$orig_at"  ]] && task_options+=" -e original_emitted_at=${orig_at}"
+
+    if [[ -n "$target_ver" ]]; then
+      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
+      task_options+=" -e target_version='${esc_tv}'"
+    fi
+  fi
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special cases: parametric reboot hours from task name
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+
+  # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4
+  if [[ "$task" =~ ^(.+)_tonight$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+
+    # now, today 01:00, tomorrow 01:00 (local time)
+    local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
+    now_s="$(date +%s)"
+    today1_s="$(date -d 'today 01:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    if (( now_s < today1_s )); then
+      next1_s="$today1_s"
+    else
+      next1_s="$tomorrow1_s"
+    fi
+    diff_s=$(( next1_s - now_s ))
+    # Ceil hours so current minutes are preserved as in your examples
+    ceil_h=$(( (diff_s + 3599) / 3600 ))
+    rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
+    total_h=$(( ceil_h + rnd ))
+
+    extra_nbplay_opts+=("-erebootin=${total_h}")
+    log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
+  elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
+    # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin=<n>
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor"
+    extra_nbplay_opts+=("-erebootin=${n}")
+    log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})"
+  fi
+
+  # --- Default rebootin for reboot-family when not explicitly provided ---
+  if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot)$ ]]; then
+    # only set if neither task_options nor extra_nbplay_opts already contain rebootin
+    if [[ "$task_options" != *"rebootin="* ]] && ! printf '%s\n' "${extra_nbplay_opts[@]}" | grep -q 'rebootin='; then
+      # restore legacy behavior: immediate reboot if none specified
+      extra_nbplay_opts+=("-erebootin=0")
+      log "No rebootin provided; defaulting to rebootin=0 for ${base_task}"
+    fi
+  fi
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env.
+  # This sets RMQ_EXCHANGE=controls only for this nbplay invocation.
+  if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide source ---
+if [[ -n "$EXCHANGE" ]]; then
+  # Mode B (unchanged): bind a temp queue to exchange+routing key
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+
+  # Single-queue consume loop (unchanged branch)
+  log "Press Ctrl+C to stop."
+  while :; do
+    RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+      "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+    }')"
+
+    if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+      sleep "$SLEEP_SECS"
+      continue
+    fi
+
+    if has_jq; then
+      echo "$RESP" | jq -c '.[]' | while read -r item; do
+        payload=$(printf '%s' "$item" | jq -r '.payload')
+        if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+          ptype="$(printf '%s' "$payload" | jq -r 'type')"
+          if [[ "$ptype" == "string" ]]; then
+            decoded="$(printf '%s' "$payload" | jq -r .)"
+            if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+              dispatch_task "$decoded"
+            else
+              warn "payload is a string but not JSON after unquote; skipping."
+            fi
+          else
+            dispatch_task "$payload"
+          fi
+        else
+          warn "payload is not JSON; skipping."
+        fi
+      done
+    else
+      warn "jq not found; cannot parse JSON payloads. Exiting."
+      exit 3
+    fi
+  done
+
+else
+  # Mode A: direct queue(s). Support CSV list in QUEUE.
+  IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE"
+  for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done
+
+  if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then
+    err "No queues configured (QUEUE env empty)."
+    exit 2
+  fi
+
+  log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)"
+  log "Press Ctrl+C to stop."
+
+  # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
+  while :; do
+    local_got_message=0
+
+    for Q in "${QUEUE_LIST[@]}"; do
+      RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{
+        "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+      }')"
+
+      if [[ "$RESP" != "[]" && -n "$RESP" ]]; then
+        local_got_message=1
+        log "Dequeued from [$Q]"
+        if has_jq; then
+          echo "$RESP" | jq -c '.[]' | while read -r item; do
+            payload=$(printf '%s' "$item" | jq -r '.payload')
+            if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+              ptype="$(printf '%s' "$payload" | jq -r 'type')"
+              if [[ "$ptype" == "string" ]]; then
+                decoded="$(printf '%s' "$payload" | jq -r .)"
+                if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+                  dispatch_task "$decoded"
+                else
+                  warn "payload is a string but not JSON after unquote; skipping."
+                fi
+              else
+                dispatch_task "$payload"
+              fi
+            else
+              warn "payload is not JSON; skipping."
+            fi
+          done
+        else
+          warn "jq not found; cannot parse JSON payloads. Exiting."
+          exit 3
+        fi
+
+        # After processing a message from this queue, start the round over (fair-ish polling).
+        break
+      fi
+    done
+
+    if [[ "$local_got_message" -eq 0 ]]; then
+      sleep "$SLEEP_SECS"
+    fi
+  done
+fi

files/rabbit-client.sh--bbz

@@ -0,0 +1,287 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Hard override as in your current file
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default). Supports CSV for multiple queues.
+# CHANGED: default now includes both normal and persuasive work queues.
+QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}"    # e.g. "queue_deviceconfig,queue_persuasive"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log()  { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
+  if [[ "$task" == "afterupgrade_check" ]]; then
+    local attempt corr_id orig_at target_ver
+    attempt="$(jq -r '.attempt // empty' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
+    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+
+    [[ -n "$attempt"  ]] && task_options+=" -e attempt=${attempt}"
+    [[ -n "$corr_id"  ]] && task_options+=" -e correlation_id=${corr_id}"
+    [[ -n "$orig_at"  ]] && task_options+=" -e original_emitted_at=${orig_at}"
+
+    if [[ -n "$target_ver" ]]; then
+      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
+      task_options+=" -e target_version='${esc_tv}'"
+    fi
+  fi
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special cases: parametric reboot hours from task name
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+
+  # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4
+  if [[ "$task" =~ ^(.+)_tonight$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+
+    # now, today 01:00, tomorrow 01:00 (local time)
+    local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
+    now_s="$(date +%s)"
+    today1_s="$(date -d 'today 01:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    if (( now_s < today1_s )); then
+      next1_s="$today1_s"
+    else
+      next1_s="$tomorrow1_s"
+    fi
+    diff_s=$(( next1_s - now_s ))
+    # Ceil hours so current minutes are preserved as in your examples
+    ceil_h=$(( (diff_s + 3599) / 3600 ))
+    rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
+    total_h=$(( ceil_h + rnd ))
+
+    extra_nbplay_opts+=("-erebootin=${total_h}")
+    log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
+  elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
+    # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin=<n>
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor"
+    extra_nbplay_opts+=("-erebootin=${n}")
+    log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})"
+  fi
+  # --- Default rebootin for reboot-family when not explicitly provided
+  if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot)$ ]]; then
+    # only set if neither task_options nor extra_nbplay_opts already contain rebootin
+    if [[ "$task_options" != *"rebootin="* ]] && ! printf '%s\n' "${extra_nbplay_opts[@]}" | grep -q 'rebootin='; then
+      # For these playbooks, rebootin is interpreted as hours; 0 == reboot now.
+      extra_nbplay_opts+=("-e" "rebootin=0")
+      log "No rebootin provided; defaulting to rebootin=0 for ${base_task}"
+    fi
+  fi
+  
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env.
+  # This sets RMQ_EXCHANGE=controls only for this nbplay invocation.
+  if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide source ---
+if [[ -n "$EXCHANGE" ]]; then
+  # Mode B (unchanged): bind a temp queue to exchange+routing key
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+
+  # Single-queue consume loop (unchanged branch)
+  log "Press Ctrl+C to stop."
+  while :; do
+    RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+      "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+    }')"
+
+    if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+      sleep "$SLEEP_SECS"
+      continue
+    fi
+
+    if has_jq; then
+      echo "$RESP" | jq -c '.[]' | while read -r item; do
+        payload=$(printf '%s' "$item" | jq -r '.payload')
+        if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+          ptype="$(printf '%s' "$payload" | jq -r 'type')"
+          if [[ "$ptype" == "string" ]]; then
+            decoded="$(printf '%s' "$payload" | jq -r .)"
+            if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+              dispatch_task "$decoded"
+            else
+              warn "payload is a string but not JSON after unquote; skipping."
+            fi
+          else
+            dispatch_task "$payload"
+          fi
+        else
+          warn "payload is not JSON; skipping."
+        fi
+      done
+    else
+      warn "jq not found; cannot parse JSON payloads. Exiting."
+      exit 3
+    fi
+  done
+
+else
+  # Mode A: direct queue(s). Support CSV list in QUEUE.
+  IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE"
+  for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done
+
+  if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then
+    err "No queues configured (QUEUE env empty)."
+    exit 2
+  fi
+
+  log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)"
+  log "Press Ctrl+C to stop."
+
+  # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
+  while :; do
+    local_got_message=0
+
+    for Q in "${QUEUE_LIST[@]}"; do
+      RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{
+        "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+      }')"
+
+      if [[ "$RESP" != "[]" && -n "$RESP" ]]; then
+        local_got_message=1
+        log "Dequeued from [$Q]"
+        if has_jq; then
+          echo "$RESP" | jq -c '.[]' | while read -r item; do
+            payload=$(printf '%s' "$item" | jq -r '.payload')
+            if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+              ptype="$(printf '%s' "$payload" | jq -r 'type')"
+              if [[ "$ptype" == "string" ]]; then
+                decoded="$(printf '%s' "$payload" | jq -r .)"
+                if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+                  dispatch_task "$decoded"
+                else
+                  warn "payload is a string but not JSON after unquote; skipping."
+                fi
+              else
+                dispatch_task "$payload"
+              fi
+            else
+              warn "payload is not JSON; skipping."
+            fi
+          done
+        else
+          warn "jq not found; cannot parse JSON payloads. Exiting."
+          exit 3
+        fi
+
+        # After processing a message from this queue, start the round over (fair-ish polling).
+        break
+      fi
+    done
+
+    if [[ "$local_got_message" -eq 0 ]]; then
+      sleep "$SLEEP_SECS"
+    fi
+  done
+fi

files/rabbit-client.sh-before_multiqueue

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Hard override as in your current file
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default)
+QUEUE="${QUEUE:-queue_deviceconfig}"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log()  { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    # We build an eval that appends parsed words to the array.
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
+  if [[ "$task" == "afterupgrade_check" ]]; then
+    local attempt corr_id orig_at target_ver
+    attempt="$(jq -r '.attempt // empty' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
+    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+
+    # Build -e flags into task_options (respect existing task_options)
+    [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}"
+    [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}"
+    [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}"
+
+    # target_version may contain spaces (e.g., "2.2.2 rev 9778")
+    # append_opts uses eval, so wrap in single quotes and escape embedded single quotes.
+    if [[ -n "$target_ver" ]]; then
+      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
+      task_options+=" -e target_version='${esc_tv}'"
+    fi
+  fi
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special case: task "update-rebootin_N" (N=0..99)
+  # If matched, run playbook "update-rebootin.yml" and append "-erebootin=N" at the end of nbplay args.
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+  if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  fi
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  # Build command array and append options (respect quoted args)
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  # Append any special-case options at the very end (e.g., -erebootin=N)
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  if ! "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide queue source ---
+if [[ -n "$EXCHANGE" ]]; then
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+else
+  log "Consuming directly from queue: $QUEUE (vhost: $VHOST)"
+fi
+
+log "Press Ctrl+C to stop."
+
+# --- Consume loop ---
+while :; do
+  RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+    "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+  }')"
+
+  # Empty array => no messages
+  if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+    sleep "$SLEEP_SECS"
+    continue
+  fi
+
+  # Process one message (array of length 1)
+  if has_jq; then
+    echo "$RESP" | jq -c '.[]' | while read -r item; do
+      payload=$(printf '%s' "$item" | jq -r '.payload')
+
+      # Decode if payload is a quoted JSON string
+      if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+        ptype="$(printf '%s' "$payload" | jq -r 'type')"
+        if [[ "$ptype" == "string" ]]; then
+          decoded="$(printf '%s' "$payload" | jq -r .)"
+          if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+            dispatch_task "$decoded"
+          else
+            warn "payload is a string but not JSON after unquote; skipping."
+          fi
+        else
+          dispatch_task "$payload"
+        fi
+      else
+        warn "payload is not JSON; skipping."
+      fi
+    done
+  else
+    # Fallback: cannot parse JSON without jq
+    warn "jq not found; cannot parse JSON payloads. Exiting."
+    exit 3
+  fi
+done

files/rabbit-client.sh-beforedelayed

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default)
+QUEUE="${QUEUE:-queue_deviceconfig}"
+
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log() { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err() { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    # We build an eval that appends parsed words to the array.
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special case: task "update-rebootin_N" (N=0..99)
+  # If matched, run playbook "update-rebootin.yml" and append "-erebootin=N" at the end of nbplay args.
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+  if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  fi
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  # Build command array and append options (respect quoted args)
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  # Append any special-case options at the very end (e.g., -erebootin=N)
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  if ! "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide queue source ---
+if [[ -n "$EXCHANGE" ]]; then
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+else
+  log "Consuming directly from queue: $QUEUE (vhost: $VHOST)"
+fi
+
+log "Press Ctrl+C to stop."
+
+# --- Consume loop ---
+while :; do
+  RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+    "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+  }')"
+
+  # Empty array => no messages
+  if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+    sleep "$SLEEP_SECS"
+    continue
+  fi
+
+  # Process one message (array of length 1)
+  if has_jq; then
+    echo "$RESP" | jq -c '.[]' | while read -r item; do
+      payload=$(printf '%s' "$item" | jq -r '.payload')
+
+      # Decode if payload is a quoted JSON string
+      if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+        ptype="$(printf '%s' "$payload" | jq -r 'type')"
+        if [[ "$ptype" == "string" ]]; then
+          decoded="$(printf '%s' "$payload" | jq -r .)"
+          if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+            dispatch_task "$decoded"
+          else
+            warn "payload is a string but not JSON after unquote; skipping."
+          fi
+        else
+          dispatch_task "$payload"
+        fi
+      else
+        warn "payload is not JSON; skipping."
+      fi
+    done
+  else
+    # Fallback: cannot parse JSON without jq
+    warn "jq not found; cannot parse JSON payloads. Exiting."
+    exit 3
+  fi
+done

files/rabbit-client.sh-beforequeuefixing

@@ -0,0 +1,241 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Hard override as in your current file
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default). Supports CSV for multiple queues.
+QUEUE="${QUEUE:-queue_deviceconfig}"    # e.g. "queue_deviceconfig,queue_persuasive"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log()  { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
+  if [[ "$task" == "afterupgrade_check" ]]; then
+    local attempt corr_id orig_at target_ver
+    attempt="$(jq -r '.attempt // empty' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
+    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+
+    [[ -n "$attempt"  ]] && task_options+=" -e attempt=${attempt}"
+    [[ -n "$corr_id"  ]] && task_options+=" -e correlation_id=${corr_id}"
+    [[ -n "$orig_at"  ]] && task_options+=" -e original_emitted_at=${orig_at}"
+
+    if [[ -n "$target_ver" ]]; then
+      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
+      task_options+=" -e target_version='${esc_tv}'"
+    fi
+  fi
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special case: task "update-rebootin_N" (N=0..99)
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+  if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  fi
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  if ! "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide source ---
+if [[ -n "$EXCHANGE" ]]; then
+  # Mode B (unchanged): bind a temp queue to exchange+routing key
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+
+  # Single-queue consume loop (unchanged branch)
+  log "Press Ctrl+C to stop."
+  while :; do
+    RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+      "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+    }')"
+
+    if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+      sleep "$SLEEP_SECS"
+      continue
+    fi
+
+    if has_jq; then
+      echo "$RESP" | jq -c '.[]' | while read -r item; do
+        payload=$(printf '%s' "$item" | jq -r '.payload')
+        if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+          ptype="$(printf '%s' "$payload" | jq -r 'type')"
+          if [[ "$ptype" == "string" ]]; then
+            decoded="$(printf '%s' "$payload" | jq -r .)"
+            if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+              dispatch_task "$decoded"
+            else
+              warn "payload is a string but not JSON after unquote; skipping."
+            fi
+          else
+            dispatch_task "$payload"
+          fi
+        else
+          warn "payload is not JSON; skipping."
+        fi
+      done
+    else
+      warn "jq not found; cannot parse JSON payloads. Exiting."
+      exit 3
+    fi
+  done
+
+else
+  # Mode A: direct queue(s). Support CSV list in QUEUE.
+  IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE"
+  for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done
+
+  if ((${#QUEUE_LIST[@]} == 0)); then
+    err "No queues configured (QUEUE env empty)."
+    exit 2
+  fi
+
+  log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)"
+  log "Press Ctrl+C to stop."
+
+  # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
+  while :; do
+    local_got_message=0
+
+    for Q in "${QUEUE_LIST[@]}"; do
+      RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{
+        "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+      }')"
+
+      if [[ "$RESP" != "[]" && -n "$RESP" ]]; then
+        local_got_message=1
+        log "Dequeued from [$Q]"
+        if has_jq; then
+          echo "$RESP" | jq -c '.[]' | while read -r item; do
+            payload=$(printf '%s' "$item" | jq -r '.payload')
+            if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+              ptype="$(printf '%s' "$payload" | jq -r 'type')"
+              if [[ "$ptype" == "string" ]]; then
+                decoded="$(printf '%s' "$payload" | jq -r .)"
+                if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+                  dispatch_task "$decoded"
+                else
+                  warn "payload is a string but not JSON after unquote; skipping."
+                fi
+              else
+                dispatch_task "$payload"
+              fi
+            else
+              warn "payload is not JSON; skipping."
+            fi
+          done
+        else
+          warn "jq not found; cannot parse JSON payloads. Exiting."
+          exit 3
+        fi
+
+        # After processing a message from this queue, start the round over (fair-ish polling).
+        break
+      fi
+    done
+
+    if [[ "$local_got_message" -eq 0 ]]; then
+      sleep "$SLEEP_SECS"
+    fi
+  done
+fi

files/rabbit-client.sh-fixing-rebootin

@@ -0,0 +1,255 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Hard override as in your current file
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default). Supports CSV for multiple queues.
+# CHANGED: default now includes both normal and persuasive work queues.
+QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}"    # e.g. "queue_deviceconfig,queue_persuasive"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log()  { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
+  if [[ "$task" == "afterupgrade_check" ]]; then
+    local attempt corr_id orig_at target_ver
+    attempt="$(jq -r '.attempt // empty' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
+    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+
+    [[ -n "$attempt"  ]] && task_options+=" -e attempt=${attempt}"
+    [[ -n "$corr_id"  ]] && task_options+=" -e correlation_id=${corr_id}"
+    [[ -n "$orig_at"  ]] && task_options+=" -e original_emitted_at=${orig_at}"
+
+    if [[ -n "$target_ver" ]]; then
+      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
+      task_options+=" -e target_version='${esc_tv}'"
+    fi
+  fi
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special cases: parametric reboot hours from task name
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+  if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor"
+    # Convert hours -> minutes for nbplay's --rebootin
+    minutes=$((10#$n * 60))
+    extra_nbplay_opts+=("--rebootin" "${minutes}")
+    log "Parsed update-indoor suffix: ${n}h -> ${minutes}m (rebootin)"
+  fi
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env.
+  # This sets RMQ_EXCHANGE=controls only for this nbplay invocation.
+  if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide source ---
+if [[ -n "$EXCHANGE" ]]; then
+  # Mode B (unchanged): bind a temp queue to exchange+routing key
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+
+  # Single-queue consume loop (unchanged branch)
+  log "Press Ctrl+C to stop."
+  while :; do
+    RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+      "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+    }')"
+
+    if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+      sleep "$SLEEP_SECS"
+      continue
+    fi
+
+    if has_jq; then
+      echo "$RESP" | jq -c '.[]' | while read -r item; do
+        payload=$(printf '%s' "$item" | jq -r '.payload')
+        if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+          ptype="$(printf '%s' "$payload" | jq -r 'type')"
+          if [[ "$ptype" == "string" ]]; then
+            decoded="$(printf '%s' "$payload" | jq -r .)"
+            if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+              dispatch_task "$decoded"
+            else
+              warn "payload is a string but not JSON after unquote; skipping."
+            fi
+          else
+            dispatch_task "$payload"
+          fi
+        else
+          warn "payload is not JSON; skipping."
+        fi
+      done
+    else
+      warn "jq not found; cannot parse JSON payloads. Exiting."
+      exit 3
+    fi
+  done
+
+else
+  # Mode A: direct queue(s). Support CSV list in QUEUE.
+  IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE"
+  for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done
+
+  if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then
+    err "No queues configured (QUEUE env empty)."
+    exit 2
+  fi
+
+  log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)"
+  log "Press Ctrl+C to stop."
+
+  # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
+  while :; do
+    local_got_message=0
+
+    for Q in "${QUEUE_LIST[@]}"; do
+      RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{
+        "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+      }')"
+
+      if [[ "$RESP" != "[]" && -n "$RESP" ]]; then
+        local_got_message=1
+        log "Dequeued from [$Q]"
+        if has_jq; then
+          echo "$RESP" | jq -c '.[]' | while read -r item; do
+            payload=$(printf '%s' "$item" | jq -r '.payload')
+            if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+              ptype="$(printf '%s' "$payload" | jq -r 'type')"
+              if [[ "$ptype" == "string" ]]; then
+                decoded="$(printf '%s' "$payload" | jq -r .)"
+                if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+                  dispatch_task "$decoded"
+                else
+                  warn "payload is a string but not JSON after unquote; skipping."
+                fi
+              else
+                dispatch_task "$payload"
+              fi
+            else
+              warn "payload is not JSON; skipping."
+            fi
+          done
+        else
+          warn "jq not found; cannot parse JSON payloads. Exiting."
+          exit 3
+        fi
+
+        # After processing a message from this queue, start the round over (fair-ish polling).
+        break
+      fi
+    done
+
+    if [[ "$local_got_message" -eq 0 ]]; then
+      sleep "$SLEEP_SECS"
+    fi
+  done
+fi

files/rabbit-client.sh-fixing-rebootin2

@@ -0,0 +1,277 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Hard override as in your current file
+RMQ_HOST="10.210.12.2"
+
+# Mode A: read from a queue (default). Supports CSV for multiple queues.
+# CHANGED: default now includes both normal and persuasive work queues.
+QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}"    # e.g. "queue_deviceconfig,queue_persuasive"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"          # empty => Mode B OFF
+ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Paths
+APP_ROOT="/opt/containers/ansible-worker/app"
+NBPLAY="${APP_ROOT}/bin/nbplay"
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; }
+
+log()  { printf '[consumer] %s\n' "$*"; }
+warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
+err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
+
+# Safely append string options into an array using eval (so quotes are honored).
+append_opts() {
+  local opts_str="$1"
+  # shellcheck disable=SC2206
+  local -n _arr_ref=$2
+  if [[ -n "$opts_str" ]]; then
+    eval '_arr_ref+=('"$opts_str"')'
+  fi
+}
+
+# Given a JSON object payload, extract fields and dispatch nbplay
+dispatch_task() {
+  local json="$1"
+  local device task playbook task_options
+  device="$(jq -er '.inscope_device // empty' <<<"$json")" || device=""
+  task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
+  task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
+
+  # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
+  if [[ "$task" == "afterupgrade_check" ]]; then
+    local attempt corr_id orig_at target_ver
+    attempt="$(jq -r '.attempt // empty' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
+    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+
+    [[ -n "$attempt"  ]] && task_options+=" -e attempt=${attempt}"
+    [[ -n "$corr_id"  ]] && task_options+=" -e correlation_id=${corr_id}"
+    [[ -n "$orig_at"  ]] && task_options+=" -e original_emitted_at=${orig_at}"
+
+    if [[ -n "$target_ver" ]]; then
+      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
+      task_options+=" -e target_version='${esc_tv}'"
+    fi
+  fi
+
+  if [[ -z "$device" || -z "$task" ]]; then
+    warn "payload missing required keys (inscope_device/task_name). Skipping."
+    return 0
+  fi
+
+  # --- Special cases: parametric reboot hours from task name
+  local base_task="$task"
+  local -a extra_nbplay_opts=()
+  local n=""
+
+  # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4
+  if [[ "$task" =~ ^(.+)_tonight$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+
+    # now, today 01:00, tomorrow 01:00 (local time)
+    local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h
+    now_s="$(date +%s)"
+    today1_s="$(date -d 'today 01:00' +%s)"
+    tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)"
+    if (( now_s < today1_s )); then
+      next1_s="$today1_s"
+    else
+      next1_s="$tomorrow1_s"
+    fi
+    diff_s=$(( next1_s - now_s ))
+    # Ceil hours so current minutes are preserved as in your examples
+    ceil_h=$(( (diff_s + 3599) / 3600 ))
+    rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
+    total_h=$(( ceil_h + rnd ))
+
+    extra_nbplay_opts+=("-erebootin=${total_h}")
+    log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
+  elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-rebootin"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
+    # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin=<n>
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor"
+    extra_nbplay_opts+=("-erebootin=${n}")
+    log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})"
+  fi
+
+  playbook="${APP_ROOT}/${base_task}.yml"
+  if [[ ! -f "$playbook" ]]; then
+    warn "playbook not found: $playbook (device='$device', task='$task'). Skipping."
+    return 0
+  fi
+
+  log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-<none>}"
+
+  local -a cmd=( "$NBPLAY" "$playbook" "$device" )
+  append_opts "$task_options" cmd
+
+  if ((${#extra_nbplay_opts[@]})); then
+    cmd+=("${extra_nbplay_opts[@]}")
+  fi
+
+  # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env.
+  # This sets RMQ_EXCHANGE=controls only for this nbplay invocation.
+  if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then
+    err "playbook failed (task='${task}', device='${device}')"
+    # keep consuming; message already acked by HTTP get endpoint
+  fi
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    log "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide source ---
+if [[ -n "$EXCHANGE" ]]; then
+  # Mode B (unchanged): bind a temp queue to exchange+routing key
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true
+  }' >/dev/null
+
+  log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  log "Consuming from bound temp queue: $QUEUE"
+
+  # Single-queue consume loop (unchanged branch)
+  log "Press Ctrl+C to stop."
+  while :; do
+    RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+      "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+    }')"
+
+    if [[ "$RESP" == "[]" || -z "$RESP" ]]; then
+      sleep "$SLEEP_SECS"
+      continue
+    fi
+
+    if has_jq; then
+      echo "$RESP" | jq -c '.[]' | while read -r item; do
+        payload=$(printf '%s' "$item" | jq -r '.payload')
+        if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+          ptype="$(printf '%s' "$payload" | jq -r 'type')"
+          if [[ "$ptype" == "string" ]]; then
+            decoded="$(printf '%s' "$payload" | jq -r .)"
+            if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+              dispatch_task "$decoded"
+            else
+              warn "payload is a string but not JSON after unquote; skipping."
+            fi
+          else
+            dispatch_task "$payload"
+          fi
+        else
+          warn "payload is not JSON; skipping."
+        fi
+      done
+    else
+      warn "jq not found; cannot parse JSON payloads. Exiting."
+      exit 3
+    fi
+  done
+
+else
+  # Mode A: direct queue(s). Support CSV list in QUEUE.
+  IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE"
+  for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done
+
+  if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then
+    err "No queues configured (QUEUE env empty)."
+    exit 2
+  fi
+
+  log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)"
+  log "Press Ctrl+C to stop."
+
+  # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
+  while :; do
+    local_got_message=0
+
+    for Q in "${QUEUE_LIST[@]}"; do
+      RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{
+        "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
+      }')"
+
+      if [[ "$RESP" != "[]" && -n "$RESP" ]]; then
+        local_got_message=1
+        log "Dequeued from [$Q]"
+        if has_jq; then
+          echo "$RESP" | jq -c '.[]' | while read -r item; do
+            payload=$(printf '%s' "$item" | jq -r '.payload')
+            if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+              ptype="$(printf '%s' "$payload" | jq -r 'type')"
+              if [[ "$ptype" == "string" ]]; then
+                decoded="$(printf '%s' "$payload" | jq -r .)"
+                if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+                  dispatch_task "$decoded"
+                else
+                  warn "payload is a string but not JSON after unquote; skipping."
+                fi
+              else
+                dispatch_task "$payload"
+              fi
+            else
+              warn "payload is not JSON; skipping."
+            fi
+          done
+        else
+          warn "jq not found; cannot parse JSON payloads. Exiting."
+          exit 3
+        fi
+
+        # After processing a message from this queue, start the round over (fair-ish polling).
+        break
+      fi
+    done
+
+    if [[ "$local_got_message" -eq 0 ]]; then
+      sleep "$SLEEP_SECS"
+    fi
+  done
+fi

files/rabbit-consumer.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# --- Config (override via env or flags) ---
+RMQ_USER="${RMQ_USER:-admin}"
+RMQ_PASS="${RMQ_PASS:-change_me}"
+RMQ_HOST="${RMQ_HOST:-localhost}"
+RMQ_PORT="${RMQ_PORT:-15672}"
+VHOST="${VHOST:-app}"
+
+# Mode A: read from a queue (default)
+QUEUE="${QUEUE:-queue1}"
+
+# Mode B: bind temp queue to an exchange + routing key and consume from it
+EXCHANGE="${EXCHANGE:-}"        # e.g. "amq.topic" or "my-exchange". Leave empty to skip binding mode.
+ROUTING_KEY="${ROUTING_KEY:-#}" # pattern for topic/direct, default is catch-all "#"
+
+# Polling interval when no messages
+SLEEP_SECS="${SLEEP_SECS:-1}"
+
+# Pretty-print with jq if available
+USE_JQ="${USE_JQ:-auto}"        # auto|yes|no
+USE_COLOR="${USE_COLOR:-yes}"   # yes|no -> 'yes' forces color with jq -C
+
+# --- Helpers ---
+api() {
+  local method="$1"; shift
+  local path="$1"; shift
+  local data="${1:-}"
+  if [[ -n "$data" ]]; then
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data"
+  else
+    curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path"
+  fi
+}
+
+has_jq() { command -v jq >/dev/null 2>&1; }
+
+# pretty(): colorized JSON when jq is present. We **force** color with -C so it survives pipes/subshells.
+pretty() {
+  if [[ "$USE_JQ" == "yes" ]] || { [[ "$USE_JQ" == "auto" ]] && has_jq; }; then
+    if [[ "$USE_COLOR" == "yes" ]]; then
+      jq -C .
+    else
+      jq .
+    fi
+  else
+    cat
+  fi
+}
+
+urlenc() {
+  # vhost is simple ("app"), so we skip full encoding, but keep function for completeness
+  printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'
+}
+
+cleanup_queue=""
+cleanup() {
+  if [[ -n "$cleanup_queue" ]]; then
+    echo "Cleaning up temp queue: $cleanup_queue"
+    api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+# --- Setup: decide queue source ---
+if [[ -n "$EXCHANGE" ]]; then
+  # Create a temp, exclusive, auto-delete queue and bind to exchange
+  cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)"
+  echo "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)"
+  api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{
+    "auto_delete": true,
+    "durable": false,
+    "arguments": {},
+    "exclusive": true
+  }' >/dev/null
+
+  echo "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'"
+  api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{
+    \"routing_key\": \"$ROUTING_KEY\",
+    \"arguments\": {}
+  }" >/dev/null
+
+  QUEUE="$cleanup_queue"
+  echo "Consuming from bound temp queue: $QUEUE"
+else
+  echo "Consuming directly from queue: $QUEUE (vhost: $VHOST)"
+fi
+
+echo "Press Ctrl+C to stop."
+
+# --- Consume loop ---
+while :; do
+  RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
+    "count": 1,
+    "ackmode": "ack_requeue_false",
+    "encoding": "auto",
+    "truncate": 1000000
+  }')"
+
+  # Empty array => no messages
+  if [[ "$RESP" == "[]" ]] || [[ -z "$RESP" ]]; then
+    sleep "$SLEEP_SECS"
+    continue
+  fi
+
+  # Print one-by-one (the API returns an array)
+  if has_jq && { [[ "$USE_JQ" == "yes" ]] || [[ "$USE_JQ" == "auto" ]]; }; then
+    echo "$RESP" | jq -c '.[]' | while read -r item; do
+      # Extract fields (raw to avoid extra quotes)
+      payload=$(printf '%s' "$item" | jq -r '.payload')
+      rk=$(printf '%s' "$item" | jq -r '.routing_key')
+      ex=$(printf '%s' "$item" | jq -r '.exchange')
+
+      echo "-----"
+      echo "exchange: ${ex:-\"\"}"
+      echo "routing_key: $rk"
+      echo "payload:"
+
+      # Smart payload handling:
+      # 1) If payload parses as JSON:
+      #    - If it's a JSON string, decode once; if decoded text is JSON, pretty-print it; else print plain text.
+      #    - If it's object/array/etc, pretty-print directly.
+      # 2) If payload isn't JSON at all, print as-is.
+      if jq -e . >/dev/null 2>&1 <<<"$payload"; then
+        ptype="$(printf '%s' "$payload" | jq -r 'type')"
+        if [[ "$ptype" == "string" ]]; then
+          decoded="$(printf '%s' "$payload" | jq -r .)"
+          if jq -e . >/dev/null 2>&1 <<<"$decoded"; then
+            printf '%s' "$decoded" | pretty
+          else
+            printf '%s\n' "$decoded"
+          fi
+        else
+          printf '%s' "$payload" | pretty
+        fi
+      else
+        printf '%s\n' "$payload"
+      fi
+    done
+  else
+    # Minimal parsing without jq
+    echo "$RESP"
+  fi
+done

files/ssh.yml

@@ -0,0 +1,3 @@
+ansible_user: root
+ansible_ssh_pass: wavewave
+