1626

files/ansible-playbooks/afterupgrade_check.yml

@@ -23,6 +23,8 @@
     # Do NOT self-reference max_attempts. We’ll normalize below.
     max_attempts_default: 3
 
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
   tasks:
     # ---- Normalize metadata safely (no self-referential defaults) ----
     - name: Normalize metadata (no clever transforms)
@@ -34,6 +36,21 @@
         target_version: "{{ target_version | default('') }}"
         # preserve the original string verbatim for all subsequent retries
         target_version_full: "{{ target_version | default('') }}"
+        is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+
+    - name: Debug is_run_by mode
+      ansible.builtin.debug:
+        msg:
+          - "is_run_by={{ is_run_by | default('UNSET') }}"
+          - "is_run_by_effective={{ is_run_by_effective }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
 
     - name: Show received metadata
       ansible.builtin.debug:
@@ -116,7 +133,65 @@
     - name: Evaluate version match (full-string contains check)
       when: nc_probe.rc == 0 and banner_probe.rc == 0
       ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and (mac_probe.rc | default(1)) == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
 
     # ---- Journaling paths ----
     # Success: banner matches expected full target_version
@@ -129,8 +204,9 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
 
     - name: Publish success journal to control queue
       when: journal_success_payload is defined
@@ -154,6 +230,118 @@
       register: rmq_j_success
       changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
 
+    - name: Success | Publish action_state done
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'done' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success | Clear action_restart_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_restart_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success | Clear action_next
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+      
+    - name: Success | Clear action_next_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+      
     # NEW: send a control tag to clean up device state on success
     - name: Build cleanup control payload (update_cleanup_success)
       when: journal_success_payload is defined
@@ -163,6 +351,7 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "update_cleanup_success"
 
+
     - name: Publish cleanup control message to control queue
       when: control_cleanup_payload is defined
       delegate_to: localhost
@@ -185,6 +374,50 @@
       register: rmq_cleanup_success
       changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
 
+    # Scheduled success only: chain next step (set action_next + trigger worker)
+    - name: Scheduled success | Set action_next to sot-updater-scheduler
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success | Publish sot-updater-scheduler work message
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
     # Mismatch: reachable & banner read, but not equal to target_version
     - name: Build mismatch journal payload
       when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
@@ -195,7 +428,7 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
 
     - name: Publish mismatch journal to control queue
@@ -220,6 +453,34 @@
       register: rmq_j_mismatch
       changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
 
+    - name: Failure | Pause before action_state failed
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_mismatch_payload is defined
+
+    - name: Failure | Publish action_state failed (mismatch)
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'failed' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     # SSH error path: TCP OK, but SSH failed
     - name: Build failure journal payload (ssh error) + mark retry
       when: nc_probe.rc == 0 and banner_probe.rc != 0
@@ -306,6 +567,34 @@
       register: rmq_j_gaveup
       changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
 
+    - name: Failure | Pause before action_state failed (gave up)
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_gaveup_payload is defined
+
+    - name: Failure | Publish action_state failed (gave up)
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'failed' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     - name: Stop host after final gave-up
       when: journal_gaveup_payload is defined
       ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_check_original.yml

@@ -0,0 +1,363 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_indoor_check.yml

@@ -344,7 +344,8 @@
       delegate_to: localhost
       ansible.builtin.shell: |
         sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          -o PreferredAuthentications=password -o PasswordAuthentication=yes -o NumberOfPasswordPrompts=1 \
+          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o ConnectTimeout=10 \
           root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
       register: dev2_fwver
       changed_when: false
@@ -370,6 +371,14 @@
         effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
         correlation_id: "{{ correlation_id | default('') }}"
         original_emitted_at: "{{ original_emitted_at | default('') }}"
+        is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+
+    - name: Debug is_run_by mode
+      ansible.builtin.debug:
+        msg:
+          - "is_run_by={{ is_run_by | default('UNSET') }}"
+          - "is_run_by_effective={{ is_run_by_effective }}"
 
     - name: Compute read_ok flag based on firmware readout
       delegate_to: localhost
@@ -584,6 +593,13 @@
       register: rmq_tag_remove_sched
       changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
 
+    - name: Success pause before NetBox wrapup custom fields 1
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
     # --- Normalize firmware string and set custom field on success ---
     - name: Capture raw firmware banner for normalization on success path
       when: journal_success_payload is defined
@@ -641,6 +657,197 @@
       register: rmq_customfield_fw
       changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
 
+    - name: Success publish action_state done
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'done' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success clear action_restart_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_restart_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success clear action_next
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success clear action_next_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Build cleanup control payload update_cleanup_success
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Scheduled success set action_next to sot-updater-scheduler
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success publish sot-updater-scheduler work message
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     # MISMATCH path
     - name: Build journal payload for version mismatch after upgrade
       when: (read_ok | bool) and (not (version_match | bool))
@@ -676,6 +883,56 @@
       register: rmq_j_mismatch
       changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
 
+    - name: Failure pause before action_state failed mismatch
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_mismatch_payload is defined
+
+    - name: Failure publish action_state failed mismatch
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'ready' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    - name: Failure publish action_next sot-updater-scheduler mismatch
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     - name: Stop host after mismatch path is handled
       when: journal_mismatch_payload is defined
       ansible.builtin.meta: end_host
@@ -754,6 +1011,56 @@
       register: rmq_pub_gaveup
       changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
 
+    - name: Failure pause before action_state failed gaveup
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_gaveup_payload is defined
+
+    - name: Failure publish action_state failed gaveup
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'ready' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    - name: Failure publish action_next sot-updater-scheduler gaveup
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     # Only schedule next attempt if budget left
     - name: Build delayed payload for next indoor attempt wait ten minutes
       when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))

files/ansible-playbooks/afterupgrade_withcurl.yml

@@ -0,0 +1,437 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+# --- Manual run defaults (so we can execute without -e) --- 
+# These are safe to leave here; anything passed via -e will still override them.
+    attempt: 1 
+    max_attempts: 3 
+    current_delay_sec: 300 
+    correlation_id: "6f680073dc7c" 
+    original_emitted_at: "2025-10-30T18:46:52Z" 
+    target_version: "2.2.3 rev 9800" 
+    # Intentionally keep this empty to exercise the target_version_effective logic. 
+    target_version_full: "" 
+    schema_version: 1
+
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/connstats-deploy.yml

@@ -0,0 +1,202 @@
+---
+- name: Deploy connstats (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    # Cron line we must ensure (preserve any other lines)
+    connstats_cron_line: "*/10 * * * * /root/connstats.sh --debug --always-find-offset  >/dev/null 2>&1"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+            ############ step 2
+
+            - name: Compute MD5 of local connstats.sh
+              delegate_to: localhost
+              command: md5sum files/connstats.sh
+              register: md5_local_cstats
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/connstats.sh
+              raw: "md5sum /root/connstats.sh || busybox md5sum /root/connstats.sh"
+              register: md5_remote_cstats
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if connstats.sh needs upload
+              set_fact:
+                upload_cstats: >-
+                  {{ (md5_remote_cstats.rc != 0)
+                     or ((md5_local_cstats.stdout.split()[0])
+                         != (md5_remote_cstats.stdout.split()[0] if (md5_remote_cstats.stdout is defined) else '')) }}
+
+            - name: Upload connstats.sh via scp (overwrite if changed)
+              when: upload_cstats | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/connstats.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/connstats.sh
+              register: scp_connstats
+              retries: 3
+              delay: 2
+              until: scp_connstats.rc == 0
+
+            - name: Ensure /root/connstats.sh is executable and owned by root
+              raw: |
+                chown root:root /root/connstats.sh && chmod 0755 /root/connstats.sh
+
+            ############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if connstats cron line already present
+              raw: |
+                grep -Eq '^\*/10[[:space:]]+\*[[:space:]]+\*[[:space:]]+\*[[:space:]]+\*[[:space:]]+/root/connstats\.sh[[:space:]]+--debug[[:space:]]+--always-find-offset([[:space:]]+>/dev/null[[:space:]]+2>&1)?[[:space:]]*$' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet connstats-crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/connstats-crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/connstats-crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/connstats-crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/connstats-crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_cstats | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # custom field update (per your sample)
+        - name: Publish custom-field update connstats deployed to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': inscope_device_name,
+                'task_name': 'custom_field_set',
+                'task_add1': 'connstats',
+                'task_result': 'deployed'
+              } | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: false
+
+        # final wrap-up journal "connstats: ..." with actions performed
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_cstats | default(false) | bool)                         | ternary(['uploaded connstats.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added connstats crontab entry'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                connstats: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/connstats-remove.yml

@@ -0,0 +1,179 @@
+---
+- name: Remove connstats (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+            ############ step 2
+
+            - name: Check if connstats cron line is present
+              raw: |
+                if [ -f /etc/crontabs/root ]; then
+                    grep -Eq '/root/connstats\.sh([[:space:]]|$)' /etc/crontabs/root
+                else
+                    exit 1
+                fi
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Remove connstats cron line (preserve any other lines)
+              when: cron_grep.rc == 0
+              raw: |
+                sed -i '\|/root/connstats\.sh|d' /etc/crontabs/root
+              register: cron_remove
+              changed_when: true
+
+            - name: Check if /root/connstats.log exists
+              raw: "test -e /root/connstats.log"
+              register: connstats_log_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Remove /root/connstats.log
+              when: connstats_log_grep.rc == 0
+              raw: "rm -f /root/connstats.log"
+              register: connstats_log_remove
+              changed_when: true
+
+            - name: Check if /root/connstats.sh exists
+              raw: "test -e /root/connstats.sh"
+              register: connstats_script_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Remove /root/connstats.sh
+              when: connstats_script_grep.rc == 0
+              raw: "rm -f /root/connstats.sh"
+              register: connstats_script_remove
+              changed_when: true
+
+            - name: Set result status (success removed or no change)
+              set_fact:
+                result_status: >-
+                  {{ 'SUCCESS_REMOVED'
+                     if ((cron_grep.rc == 0)
+                         or (connstats_log_grep.rc == 0)
+                         or (connstats_script_grep.rc == 0))
+                     else 'SUCCESS_NO_CHANGE' }}
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # custom field update (per your sample)
+        - name: Publish custom-field update connstats removed to control queue
+          when: result_status == 'SUCCESS_REMOVED' or result_status == 'SUCCESS_NO_CHANGE'
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': inscope_device_name,
+                'task_name': 'custom_field_set',
+                'task_add1': 'connstats',
+                'task_result': 'removed'
+              } | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: false
+
+        # final wrap-up journal "connstats: ..." with actions performed
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + (((cron_grep is defined) and ((cron_grep.rc | default(1)) == 0))                  | ternary(['removed connstats crontab entry'], []))
+                + (((connstats_log_grep is defined) and ((connstats_log_grep.rc | default(1)) == 0)) | ternary(['removed connstats.log'], []))
+                + (((connstats_script_grep is defined) and ((connstats_script_grep.rc | default(1)) == 0)) | ternary(['removed connstats.sh'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                connstats: {{ 'success' if (result_status == 'SUCCESS_REMOVED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/connstats-scheduler-remover.yml

@@ -0,0 +1,244 @@
+# connstats-scheduler-remover.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run connstats-remove.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: connstats-remove.yml
+  vars:
+    is_run_by: "scheduler"
+
+
+- name: Phase 4 | NetBox wrapup and chaining for connstats removal
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: Wrapper compute connstats removal outcome
+      set_fact:
+        inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+        connstats_ok: "{{ (result_status | default('') | regex_search('^SUCCESS_')) is not none }}"
+        connstats_status: "{{ result_status | default('UNKNOWN') }}"
+      changed_when: false
+
+    - name: Wrapper pause before clear action_next_timestamp
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper clear action_next_timestamp
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before set action_state
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_state
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': (connstats_ok | ternary('done','failed')) } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before chaining to sot updater
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_next to sot-updater-scheduler
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper publish sot-updater-scheduler work message
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before final journal
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper final journal report
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'journal_add', 'task_result': (connstats_ok | ternary('connstats: successfully removed','connstats: removal failed')) ~ ' (' ~ connstats_status ~ ')' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/connstats-scheduler.yml

@@ -0,0 +1,244 @@
+# connstats-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run connstats-deploy.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: connstats-deploy.yml
+  vars:
+    is_run_by: "scheduler"
+
+
+- name: Phase 4 | NetBox wrapup and chaining for connstats
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: Wrapper compute connstats outcome
+      set_fact:
+        inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+        connstats_ok: "{{ (result_status | default('') | regex_search('^SUCCESS_')) is not none }}"
+        connstats_status: "{{ result_status | default('UNKNOWN') }}"
+      changed_when: false
+
+    - name: Wrapper pause before clear action_next_timestamp
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper clear action_next_timestamp
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before set action_state
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_state
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': (connstats_ok | ternary('done','failed')) } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before chaining to sot updater
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_next to sot-updater-scheduler
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper publish sot-updater-scheduler work message
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before final journal
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper final journal report
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'journal_add', 'task_result': (connstats_ok | ternary('connstats: successfully deployed','connstats: deployment failed')) ~ ' (' ~ connstats_status ~ ')' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/indoor-restart-cloudagent.yml

@@ -0,0 +1,714 @@
+- name: Indoor DEV2 cloud-agent bounce via DEV1 → LLDP/tunnel → DEV2 (connection logic preserved; bootenv removed)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Busybox-safe PATH prefix for all remote raw calls on DEV1
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # DEV1 credentials (stable, like rebootin222)
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    # Tunnel target DEV2 behind DEV1
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    # Temp IP we add to DEV1 so it can reach DEV2
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+
+    # DEV2 behind the tunnel (or reachable directly via LLDP 10.x)
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # SSH options used from controller
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
+
+    # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # ---------------- Debugging toggle ----------------
+    debugging: true
+
+    # ---------------- DEV2 connection (decided early) ----------------
+    # "tunnel" (default) or "direct_lldp"
+    dev2_conn_method: "tunnel"
+    dev2_ssh_host: ""
+    dev2_ssh_port: ""
+
+  pre_tasks:
+    # ------------------------------- Hostname sanity DEV1 -------------------------------
+    - name: Read DEV1 hostname (busybox-safe)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Debug incoming parameters and defaults
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "rmq_host={{ rmq_host }}"
+          - "rmq_port={{ rmq_port }}"
+          - "rmq_vhost={{ rmq_vhost }}"
+          - "rmq_exchange={{ rmq_exchange }}"
+          - "control_queue={{ control_queue }}"
+
+    - name: Stop early if connected DEV1 hostname != inventory
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    # ============================ LLDP-FIRST CONNECTION DECISION ============================
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+    - name: Discover DEV2 candidate IP via LLDP on DEV1
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP (if any)
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP candidate range
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidate and classification
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP candidate IP={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP class={{ lldp_ip_class | default('none') }}"
+
+    # -------------------- CHANGE 1: override tunnel target from LLDP for 192.168.x.x --------------------
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x (for tunnel target)
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method to tunnel by default
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "tunnel"
+      changed_when: false
+
+    - name: Switch to direct LLDP mode for 10.x.x.x
+      when: (lldp_ip_class | trim) == "10"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "direct_lldp"
+      changed_when: false
+
+    - name: Debug connection method decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_method={{ dev2_conn_method }}"
+          - "lldp_dev2_ip={{ lldp_dev2_ip | default('<none>') }}"
+
+    # ============================ DIRECT LLDP AUTH (10.x) ============================
+    - name: Try DEV2 login via direct LLDP IP with 'basicpass' (10.x)
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass' for direct LLDP if previous login succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IP with 'basicpass2' (10.x, only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used is not defined)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass2' for direct LLDP if second login succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Mark DEV2 auth as NONE for direct LLDP if both attempts failed
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    - name: Set direct LLDP DEV2 SSH host/port (if auth succeeded)
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_ssh_host: "{{ lldp_dev2_ip }}"
+        dev2_ssh_port: 22
+      changed_when: false
+
+    # ============================ TUNNEL PREP (DEV1 temp IP + tunnel) ============================
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0)
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+            for i in $(seq 1 25); do
+                p="$(shuf -i 20000-39999 -n 1)"
+                if command -v ss >/dev/null 2>&1; then
+                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                        echo "$p"; return 0
+                    fi
+                else
+                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                        echo "$p"; return 0
+                    fi
+                fi
+            done
+            return 1
+        }
+        pick
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Record chosen local port and create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build path for SSH ControlMaster socket
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+            -M -S "{{ _ctrl_sock }}" \
+            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+            "${USER}@${HOST}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass' if previous login succeeded (tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass2' if second login succeeded (tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Mark DEV2 auth as NONE if both attempts failed (tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    - name: Set DEV2 SSH host/port for tunnel mode (if auth succeeded)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_ssh_host: "127.0.0.1"
+        dev2_ssh_port: "{{ _local_port }}"
+      changed_when: false
+
+    # -------------------- CHANGE 2: safety guard using dev2_mac (only if we actually discovered one) --------------------
+    - name: Read remote eth0 MAC via selected connection (guard ensure this is DEV2)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE" and (dev2_mac | default('') | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "cat /sys/class/net/eth0/address 2>/dev/null || echo"
+      args:
+        executable: /bin/bash
+      register: dev2_eth0_mac_read
+      changed_when: false
+
+    - name: Abort if remote eth0 MAC != discovered DEV2 MAC
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE" and (dev2_mac | default('') | trim | length > 0) and ((dev2_eth0_mac_read.stdout | default('') | trim | lower) != (dev2_mac | trim | lower))
+      ansible.builtin.fail:
+        msg: >
+          Safety stop: tunnel reached wrong device.
+          expected_dev2_mac={{ dev2_mac | trim }},
+          remote_eth0_mac={{ dev2_eth0_mac_read.stdout | default('') | trim }}.
+
+    # ============================ DEV2 HOSTNAME GUARD ============================
+    - name: Stop and journal if DEV2 auth failed (no passfile worked)
+      when: dev2_passfile_used == "NONE"
+      block:
+        - name: Build control queue payload for indoor aborted journal (auth failure)
+          ansible.builtin.set_fact:
+            journal_indoor_aborted:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                indoor: aborted: DEV2 auth failed (basicpass/basicpass2 did not work). conn_method={{ dev2_conn_method }}
+          delegate_to: localhost
+
+        - name: Publish indoor aborted journal (auth failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_indoor_aborted | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_indoor_aborted_auth_resp
+          changed_when: (rmq_journal_indoor_aborted_auth_resp.json is defined) and (rmq_journal_indoor_aborted_auth_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_indoor_aborted_auth_resp.status != 200) or
+            (rmq_journal_indoor_aborted_auth_resp.json is not defined) or
+            (not (rmq_journal_indoor_aborted_auth_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop host after DEV2 auth failure
+          ansible.builtin.meta: end_host
+
+    - name: Read DEV2 hostname via selected connection (busybox-safe)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
+      args:
+        executable: /bin/bash
+      register: dev2_host_read
+      changed_when: false
+
+    - name: Normalize hostnames for strict compare (inventory/DEV1/DEV2)
+      ansible.builtin.set_fact:
+        _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+
+    - name: Guard DEV2 hostname must equal inventory AND DEV1 (prevents IP churn mistakes)
+      block:
+        - name: Fail if DEV2 hostname differs from inventory/DEV1
+          ansible.builtin.fail:
+            msg: >
+              Hostname mismatch: DEV2='{{ _dev2_hn }}',
+              inventory='{{ _inv_hn }}',
+              DEV1='{{ _dev1_hn }}'
+          when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
+      rescue:
+        - name: Build control queue payload for indoor aborted journal (hostname mismatch)
+          ansible.builtin.set_fact:
+            journal_indoor_aborted:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                indoor: aborted: Hostname mismatch: DEV2={{ _dev2_hn }}, inventory={{ _inv_hn }}, DEV1={{ _dev1_hn }}
+          delegate_to: localhost
+
+        - name: Publish indoor aborted journal (hostname mismatch)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_indoor_aborted | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_indoor_aborted_hn_resp
+          changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_indoor_aborted_hn_resp.status != 200) or
+            (rmq_journal_indoor_aborted_hn_resp.json is not defined) or
+            (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop host after hostname mismatch
+          ansible.builtin.meta: end_host
+
+    # ============================ JOURNAL: START ============================
+    - name: Build control queue payload for 'indoor start' journal (cloud-agent bounce)
+      ansible.builtin.set_fact:
+        journal_indoor_start:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: DEV2 reachable and hostname verified; starting cloud-agent bounce.
+            conn_method={{ dev2_conn_method }},
+            dev2={{ dev2_ssh_host }}:{{ dev2_ssh_port }}
+      delegate_to: localhost
+
+    - name: Publish 'indoor start' journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_start | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_start_resp
+      changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_start_resp.status != 200) or
+        (rmq_journal_indoor_start_resp.json is not defined) or
+        (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    # ============================ CLOUD-AGENT BOUNCE (DEV2) ============================
+    - name: Move /tmp/launchd/services/cloud-agent to /root/ on DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "set -e; PATH=/sbin:/usr/sbin:/bin:/usr/bin:\$PATH; mv -f /tmp/launchd/services/cloud-agent /root/"
+      args:
+        executable: /bin/bash
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 3
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/ on DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "set -e; PATH=/sbin:/usr/sbin:/bin:/usr/bin:\$PATH; mv -f /root/cloud-agent /tmp/launchd/services/"
+      args:
+        executable: /bin/bash
+      register: move_back
+      changed_when: true
+
+    - name: Build 'indoor updated' journal payload (cloud-agent bounced)
+      ansible.builtin.set_fact:
+        journal_indoor_updated:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: cloud-agent bounced successfully.
+            conn_method={{ dev2_conn_method }},
+            move_out_rc={{ move_out.rc | default('NA') }},
+            move_back_rc={{ move_back.rc | default('NA') }}
+      delegate_to: localhost
+
+    - name: Publish 'indoor updated' journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_updated | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_updated_resp
+      changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_updated_resp.status != 200) or
+        (rmq_journal_indoor_updated_resp.json is not defined) or
+        (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    # ---------------------------- Final operator summary (one-liners) ----------------------------
+    - name: Summary key outcomes (one-liners)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_method={{ dev2_conn_method }}"
+          - "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
+          - "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
+          - "dev2_passfile_used={{ dev2_passfile_used }}"
+          - "dev2_hostname={{ _dev2_hn | default('') }}"
+          - "bounce_move_out_rc={{ move_out.rc | default('NA') }}"
+          - "bounce_move_back_rc={{ move_back.rc | default('NA') }}"
+
+  post_tasks:
+    - name: Cleanup (always)
+      block:
+        - ansible.builtin.debug:
+            msg: "Entering cleanup block"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close SSH ControlMaster (best-effort)
+          when: dev2_conn_method == "tunnel"
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove tunnel control dir (best-effort)
+          when: dev2_conn_method == "tunnel"
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+          when: dev2_conn_method == "tunnel"
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          changed_when: del_ip.rc == 0
+          failed_when: >
+            del_ip.rc != 0
+            and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+            and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
+
+    - name: Debug temp IP removal result
+      when: dev2_conn_method == "tunnel" and del_ip is defined
+      ansible.builtin.debug:
+        msg:
+          - "del_ip.rc={{ del_ip.rc | default('') }}"
+          - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
+          - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"

files/ansible-playbooks/ptsd-migrate-wrapper.yml

@@ -0,0 +1,72 @@
+# ptsd-migrate-wrapper.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run full ptsd-migrate.yml unchanged
+
+- hosts: all
+  gather_facts: no
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('(?:^|\\s)ip=([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)(?:\\s|$)', '\\1') | default('') }}"
+        _ip: >-
+          {%- if _ip_raw is string -%}
+            {{ _ip_raw }}
+          {%- elif _ip_raw is sequence and (_ip_raw | length) > 0 -%}
+            {{ _ip_raw[0] }}
+          {%- else -%}
+            ""
+          {%- endif -%}
+      set_fact:
+        wrapper_ok_line: "{{ _ok_line }}"
+        wrapper_fail_line: "{{ _fail_line }}"
+        wrapper_nb_ip: "{{ _ip | trim }}"
+
+    - name: "Wrapper | Debug extracted values"
+      ansible.builtin.debug:
+        msg:
+          - "wrapper_ok_line='{{ wrapper_ok_line }}'"
+          - "wrapper_fail_line='{{ wrapper_fail_line | default('') }}'"
+          - "wrapper_nb_ip='{{ wrapper_nb_ip }}'"
+
+    - name: "Wrapper | Stop host if cloud reports no fresh IP"
+      when: (wrapper_fail_line | default('') | length) > 0
+      meta: end_host
+
+    - name: "Wrapper | Assert IP extracted successfully"
+      ansible.builtin.assert:
+        that:
+          - nb_script is defined
+          - nb_script.stat.exists | default(false)
+          - (wrapper_ok_line | length) > 0
+          - (wrapper_nb_ip | length) > 0
+        fail_msg: >-
+          Wrapper stop: could not extract IP from nb_onedevice_update.py output.
+          ok_line='{{ wrapper_ok_line | default('') }}'
+          fail_line='{{ wrapper_fail_line | default('') }}'
+          stdout_tail={{ (nb_preflight.stdout_lines | default([]))[-12:] }}
+
+    - name: "Wrapper | Override ansible_host to refreshed IP ({{ wrapper_nb_ip }})"
+      set_fact:
+        ansible_host: "{{ wrapper_nb_ip }}"
+
+    - name: "Wrapper | Reset SSH connection context (ensure new ansible_host is used)"
+      meta: reset_connection
+
+
+- import_playbook: ptsd-migrate.yml

files/ansible-playbooks/ptsd_reacquire_attempt-old.yml

@@ -0,0 +1,759 @@
+# ptsd_reacquire_attempt.yml
+# PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check + case A cleanup
+
+- name: "PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+  pre_tasks:
+    - name: Initialize passfile facts defensively (avoid undefined vars later)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    - name: Read DEV1 hostname (busybox-safe)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop early if connected DEV1 hostname != inventory (guard)
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+ #   - name: Restart LLDP on DEV1 to refresh DEV2 management data
+ #     ansible.builtin.raw: >
+ #       {{ pathprefix }}
+ #       mv /tmp/launchd/services/lldp-server /root/lldp-server;
+ #       sleep 2;
+ #       mv /root/lldp-server /tmp/launchd/services/lldp-server;
+ #       sleep 10
+ #     ignore_errors: true
+
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port to DEV2 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable:
+          tunnel={{ dev2_passfile_used_tunnel | default('n/a') }},
+          direct={{ dev2_passfile_used_direct | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} forwarded 127.0.0.1:{{ _local_port | default('na') }}"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for verification commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR DEV2_CMD empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            *)
+              echo "ERROR unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+    - name: VERIFY DEV2 check ppp0 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show ppp0 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_ppp0_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug ppp0 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe ppp0 exists raw={{ (dev2_ppp0_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe ppp0 exists rc={{ dev2_ppp0_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show eth0.4000 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 link probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 exists raw={{ (dev2_eth04000_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 exists rc={{ dev2_eth04000_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 has IPv4?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip -4 addr show dev eth0.4000 2>/dev/null | grep -m1 'inet ' >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_has_ipv4
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 IPv4 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 has IPv4 raw={{ (dev2_eth04000_has_ipv4.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 has IPv4 rc={{ dev2_eth04000_has_ipv4.rc | default('n/a') }}"
+
+    - name: VERIFY Summarize DHCP migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        ppp0_exists: "{{ (dev2_ppp0_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_exists: "{{ (dev2_eth04000_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_has_ipv4: "{{ (dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES' }}"
+        dhcp_migrated_ok: >-
+          {{
+            ((dev2_ppp0_exists.stdout | default('') | trim) != 'YES')
+            and
+            ((dev2_eth04000_exists.stdout | default('') | trim) == 'YES')
+            and
+            ((dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES')
+          }}
+      changed_when: false
+
+    - name: VERIFY Classify migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dhcp_state: >-
+          {% if (dhcp_migrated_ok | bool) %}
+          case_a_dhcp_ok
+          {% elif (ppp0_exists | bool) and (not (eth04000_exists | bool)) %}
+          case_b_pppoe_old
+          {% elif (ppp0_exists | bool) and (eth04000_exists | bool) %}
+          case_c_mixed
+          {% else %}
+          unknown
+          {% endif %}
+      changed_when: false
+
+    - name: VERIFY Report migration state summary
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 connectivity via {{ dev2_conn_final }}"
+          - "DEV2 probes ppp0={{ ppp0_exists }} eth0.4000={{ eth04000_exists }} eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "DEV2 DHCP migrated ok={{ dhcp_migrated_ok }}"
+
+    - name: VERIFY Report why migration is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 is not confirmed as DHCP-migrated"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "Expected for success case A: ppp0 absent, eth0.4000 present, eth0.4000 has IPv4"
+          - "Observed: ppp0={{ ppp0_exists }}, eth0.4000={{ eth04000_exists }}, eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+      changed_when: false
+
+    - name: VERIFY End host if DHCP migration state is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      ansible.builtin.meta: end_host
+
+    - name: CASE A DEV2 run config-test check before reboot cancellation
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="/usr/sbin/config-test.lua -c >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: config_test_check
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A set config-test check status
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        config_test_ok: "{{ (config_test_check.stdout | default('') | trim) == 'OK' }}"
+      changed_when: false
+
+    - name: CASE A append config-test check result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            (cleanup_report | default('case A observed'))
+            ~ ', '
+            ~ (
+              'config-test -c ok'
+              if (config_test_ok | bool)
+              else 'config-test -c failed'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A Initialize cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: "case A observed"
+      changed_when: false
+
+    - name: CASE A DEV2 find if pending reboot is present
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && echo FOUND || echo NOT_FOUND"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_present
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A DEV2 cancel reboot if present
+      when: dhcp_migrated_ok | bool and (config_test_ok | bool) and (reboot_present.stdout | default('') | trim) == "FOUND"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && killall reboot 2>/dev/null || true; echo DONE"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_cancel
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append reboot cancellation result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'pending reboot found and cancelled'
+              if (reboot_present.stdout | default('') | trim) == 'FOUND'
+              else 'no pending reboot'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 touch confirming_success marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="touch /tmp/confirming_success >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: confirming_success_touch
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append confirming_success marker result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'touched /tmp/confirming_success'
+              if (confirming_success_touch.stdout | default('') | trim) == 'OK'
+              else 'failed to touch /tmp/confirming_success'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A sleep 5 seconds
+      when: dhcp_migrated_ok | bool
+      ansible.builtin.pause:
+        seconds: 5
+
+    - name: CASE A DEV2 move config test to config with dhcp
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="mv /tmp/config-test.json /tmp/config-with-dhcp.json >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: mv_config_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append config move result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'moved /tmp/config-test.json to /tmp/config-with-dhcp.json'
+              if (mv_config_result.stdout | default('') | trim) == 'OK'
+              else 'failed to move /tmp/config-test.json to /tmp/config-with-dhcp.json'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 remove ptsd inprogress marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="rm -f /tmp/ptsd.inprogress >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: rm_inprogress_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append ptsd inprogress removal result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'removed /tmp/ptsd.inprogress'
+              if (rm_inprogress_result.stdout | default('') | trim) == 'OK'
+              else 'failed to remove /tmp/ptsd.inprogress'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A final cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ cleanup_report }}"
+      changed_when: false
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup best-effort, closing tunnel, removing temp IP, removing staged passfiles"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/ptsd_reacquire_attempt.yml

@@ -0,0 +1,973 @@
+# ptsd_reacquire_attempt.yml
+# PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check + case A cleanup
+
+- name: "PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=15
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+  pre_tasks:
+    - name: Initialize passfile facts defensively (avoid undefined vars later)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_tunnel6: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    - name: Read DEV1 hostname (busybox-safe)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop early if connected DEV1 hostname != inventory (guard)
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+ #   - name: Restart LLDP on DEV1 to refresh DEV2 management data
+ #     ansible.builtin.raw: >
+ #       {{ pathprefix }}
+ #       mv /tmp/launchd/services/lldp-server /root/lldp-server;
+ #       sleep 2;
+ #       mv /root/lldp-server /tmp/launchd/services/lldp-server;
+ #       sleep 10
+ #     ignore_errors: true
+
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep -Ei "ikeja${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Debug raw DEV2 LLDP IPv6 command output
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        var: dev2_lldp_ip6_raw.stdout_lines      
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep -Ei "ikeja${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Debug captured DEV2 LLDP IPv6 fact
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_lldp_ip6_raw={{ dev2_lldp_ip6_raw.stdout | default('') | trim }}"
+          - "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('') | trim }}"
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port to DEV2 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    - name: Decide if IPv6 tunnel fallback should be attempted
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        try_ipv6_tunnel: >-
+          {{
+            (lldp_dev2_ip6 | default('') | trim | length > 0)
+            and
+            (
+              (
+                (lldp_ip_class | trim) == '10'
+                and
+                (dev2_passfile_used_direct | default('NONE')) == 'NONE'
+              )
+              or
+              (
+                (lldp_ip_class | trim) == '192_168'
+                and
+                (dev2_passfile_used_tunnel | default('NONE')) == 'NONE'
+              )
+            )
+          }}
+      changed_when: false
+
+    - name: Pick a free local TCP port for the IPv6 tunnel (controller side)
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port6
+      changed_when: false
+
+    - name: Stop if no free local port was found for IPv6 tunnel
+      when: try_ipv6_tunnel | bool and (pick_port6.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for IPv6 tunnel SSH ControlMaster
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir6
+      changed_when: false
+
+    - name: Record chosen local port and build IPv6 tunnel ControlMaster socket path
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port6: "{{ pick_port6.stdout | trim }}"
+        _ctrl_dir6: "{{ mktemp_dir6.stdout | trim }}"
+        _ctrl_sock6: "{{ (mktemp_dir6.stdout | trim) }}/ssh_tunnel_ctl6"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port to DEV2 IPv6 22 via DEV1
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock6 }}" \
+          -L "127.0.0.1:{{ _local_port6 }}:[{{ lldp_dev2_ip6 }}%{{ dev1_iface }}]:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel6
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the IPv6 tunnel (nc)
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port6 }}"
+      args: { executable: /bin/bash }
+      register: nc_probe6
+      changed_when: false
+      ignore_errors: true
+
+    - name: Pick DEV2 password for root (IPv6 tunnel) try basicpass
+      when: try_ipv6_tunnel | bool and (nc_probe6.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port6 }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass6
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if IPv6 tunnel login succeeded
+      when: try_ipv6_tunnel | bool and (dev2_try_basicpass6.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel6: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through IPv6 tunnel with basicpass2 (only if first failed)
+      when: try_ipv6_tunnel | bool and (dev2_passfile_used_tunnel6 == "NONE") and (nc_probe6.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port6 }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass26
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if IPv6 tunnel login succeeded
+      when: try_ipv6_tunnel | bool and dev2_passfile_used_tunnel6 == "NONE" and (dev2_try_basicpass26.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel6: "basicpass2"
+      changed_when: false
+
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > tunnel6 > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- elif (dev2_passfile_used_tunnel6 | default('NONE')) != 'NONE' -%}
+            tunnel6
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable:
+          tunnel={{ dev2_passfile_used_tunnel | default('n/a') }},
+          tunnel6={{ dev2_passfile_used_tunnel6 | default('n/a') }},
+          direct={{ dev2_passfile_used_direct | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "tunnel6 passfile={{ dev2_passfile_used_tunnel6 | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} forwarded 127.0.0.1:{{ _local_port | default('na') }}"
+          - "tunnel6 target=[{{ lldp_dev2_ip6 | default('') }}%{{ dev1_iface }}]:{{ dev2_port }} forwarded 127.0.0.1:{{ _local_port6 | default('na') }}"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for verification commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR DEV2_CMD empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            tunnel6)
+              PORT="{{ _local_port6 | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel6 }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            *)
+              echo "ERROR unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+    - name: VERIFY DEV2 check ppp0 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show ppp0 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_ppp0_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug ppp0 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe ppp0 exists raw={{ (dev2_ppp0_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe ppp0 exists rc={{ dev2_ppp0_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show eth0.4000 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 link probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 exists raw={{ (dev2_eth04000_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 exists rc={{ dev2_eth04000_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 has IPv4?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip -4 addr show dev eth0.4000 2>/dev/null | grep -m1 'inet ' >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_has_ipv4
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 IPv4 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 has IPv4 raw={{ (dev2_eth04000_has_ipv4.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 has IPv4 rc={{ dev2_eth04000_has_ipv4.rc | default('n/a') }}"
+
+    - name: VERIFY Summarize DHCP migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        ppp0_exists: "{{ (dev2_ppp0_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_exists: "{{ (dev2_eth04000_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_has_ipv4: "{{ (dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES' }}"
+        dhcp_migrated_ok: >-
+          {{
+            ((dev2_ppp0_exists.stdout | default('') | trim) != 'YES')
+            and
+            ((dev2_eth04000_exists.stdout | default('') | trim) == 'YES')
+            and
+            ((dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES')
+          }}
+      changed_when: false
+
+    - name: VERIFY Classify migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dhcp_state: >-
+          {% if (dhcp_migrated_ok | bool) %}
+          case_a_dhcp_ok
+          {% elif (ppp0_exists | bool) and (not (eth04000_exists | bool)) %}
+          case_b_pppoe_old
+          {% elif (ppp0_exists | bool) and (eth04000_exists | bool) %}
+          case_c_mixed
+          {% else %}
+          unknown
+          {% endif %}
+      changed_when: false
+
+    - name: VERIFY Report migration state summary
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 connectivity via {{ dev2_conn_final }}"
+          - "DEV2 probes ppp0={{ ppp0_exists }} eth0.4000={{ eth04000_exists }} eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "DEV2 DHCP migrated ok={{ dhcp_migrated_ok }}"
+
+    - name: VERIFY Report why migration is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 is not confirmed as DHCP-migrated"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "Expected for success case A: ppp0 absent, eth0.4000 present, eth0.4000 has IPv4"
+          - "Observed: ppp0={{ ppp0_exists }}, eth0.4000={{ eth04000_exists }}, eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+      changed_when: false
+
+    - name: VERIFY End host if DHCP migration state is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      ansible.builtin.meta: end_host
+
+    - name: CASE A DEV2 run config-test check before reboot cancellation
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="/usr/sbin/config-test.lua -c >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: config_test_check
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A set config-test check status
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        config_test_ok: "{{ (config_test_check.stdout | default('') | trim) == 'OK' }}"
+      changed_when: false
+
+    - name: CASE A append config-test check result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            (cleanup_report | default('case A observed'))
+            ~ ', '
+            ~ (
+              'config-test -c ok'
+              if (config_test_ok | bool)
+              else 'config-test -c failed'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A Initialize cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: "case A observed"
+      changed_when: false
+
+    - name: CASE A DEV2 find if pending reboot is present
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && echo FOUND || echo NOT_FOUND"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_present
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A DEV2 cancel reboot if present
+      when: dhcp_migrated_ok | bool and (config_test_ok | bool) and (reboot_present.stdout | default('') | trim) == "FOUND"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && killall reboot 2>/dev/null || true; echo DONE"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_cancel
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append reboot cancellation result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'pending reboot found and cancelled'
+              if (reboot_present.stdout | default('') | trim) == 'FOUND'
+              else 'no pending reboot'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 touch confirming_success marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="touch /tmp/confirming_success >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: confirming_success_touch
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append confirming_success marker result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'touched /tmp/confirming_success'
+              if (confirming_success_touch.stdout | default('') | trim) == 'OK'
+              else 'failed to touch /tmp/confirming_success'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A sleep 5 seconds
+      when: dhcp_migrated_ok | bool
+      ansible.builtin.pause:
+        seconds: 5
+
+    - name: CASE A DEV2 move config test to config with dhcp
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="mv /tmp/config-test.json /tmp/config-with-dhcp.json >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: mv_config_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append config move result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'moved /tmp/config-test.json to /tmp/config-with-dhcp.json'
+              if (mv_config_result.stdout | default('') | trim) == 'OK'
+              else 'failed to move /tmp/config-test.json to /tmp/config-with-dhcp.json'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 remove ptsd inprogress marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="rm -f /tmp/ptsd.inprogress >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: rm_inprogress_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append ptsd inprogress removal result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'removed /tmp/ptsd.inprogress'
+              if (rm_inprogress_result.stdout | default('') | trim) == 'OK'
+              else 'failed to remove /tmp/ptsd.inprogress'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A final cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ cleanup_report }}"
+      changed_when: false
+
+    - name: CASE A publish cumulative cleanup report journal
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (cleanup_report | default(''))
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup best-effort, closing tunnel, removing temp IP, removing staged passfiles"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster for IPv6 tunnel (best-effort)
+      when: _ctrl_sock6 is defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock6 | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove IPv6 tunnel control dir (best-effort)
+      when: _ctrl_dir6 is defined
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir6 | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/redirect_and_block.yml

@@ -0,0 +1,773 @@
+# redirect_and_block.yml — connectivity + stage block scripts
+# Phase 0: reliable DEV2 connectivity + wrapper build
+# Phase 1: copy startblock.sh and stopblock.sh to /root on DEV2
+
+- name: "Redirect and block | Connectivity + stage scripts"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+    block_src_dir: "/opt/containers/ansible-worker/app"
+    block_dst_dir: "/root"
+    block_files:
+      - "startblock.sh"
+      - "stopblock.sh"
+
+  tasks:
+    - name: Initialize passfile facts (avoid undefined vars in later templates)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    # ============================ PHASE 0: LLDP DISCOVERY (IPv4 + IPv6) ============================
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    # ============================ PHASE 0: PRIMARY PATH DECISION ============================
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    # ============================ PHASE 0: DIRECT LLDP IPv4 AUTH (10.x) ============================
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ============================
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1 local_port to DEV2 port 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: SAFETY GUARD (MAC CHECK) ============================
+    - name: Read remote eth0 MAC via tunnel (guard, sanitized)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used_tunnel }}" ssh {{ ssh_opts_common }} \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /sys/class/net/eth0/address 2>/dev/null || ip link show eth0 2>/dev/null" 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_eth0_mac_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Normalize remote eth0 MAC (extract last MAC-like token)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        remote_eth0_mac: >-
+          {{
+            (
+              (dev2_eth0_mac_raw.stdout | default('') | regex_replace('\r','')) ~ "\n" ~
+              (dev2_eth0_mac_raw.stderr | default('') | regex_replace('\r',''))
+            )
+            | regex_findall('([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})')
+            | last
+            | default('')
+            | lower
+          }}
+      changed_when: false
+
+    - name: Abort if remote eth0 MAC != discovered DEV2 MAC
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+        - (remote_eth0_mac | default('') | length) > 0
+        - (remote_eth0_mac | lower) != (dev2_mac | lower)
+      ansible.builtin.fail:
+        msg: >-
+          Safety stop: tunnel reached wrong device.
+          expected_dev2_mac={{ dev2_mac }}, remote_eth0_mac={{ remote_eth0_mac }}
+
+    # ============================ PHASE 0: FALLBACKS (only if tunnel auth failed) ============================
+    - name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if timeout 20s sshpass -f "$f" ssh {{ ssh_opts_common }} \
+              -o AddressFamily=inet \
+              -o ConnectTimeout=10 \
+              "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp4
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv4 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp4: "{{ (dev2_auth_lldp4.rc == 0) | ternary((dev2_auth_lldp4.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    - name: Copy DEV2 passfiles to DEV1 for IPv6 nested SSH (last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      ansible.builtin.copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        mode: "0600"
+      loop: "{{ dev2_passfiles }}"
+      ignore_errors: true
+
+    - name: Try DEV2 login via LLDP IPv6 (nested SSH through DEV1; last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        IP6="{{ lldp_dev2_ip6 }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+              "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+              "timeout {{ ssh_timeout }}s sshpass -f '/tmp/${f}' ssh \
+                 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                 -o PubkeyAuthentication=no \
+                 -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                 -o ConnectTimeout={{ ssh_timeout }} \
+                 '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' 'echo OK' " \
+              >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp6
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv6 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp6: "{{ (dev2_auth_lldp6.rc == 0) | ternary((dev2_auth_lldp6.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    # ============================ PHASE 0: FINAL DECISION + WRAPPER FACTS ============================
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- elif (dev2_passfile_used_lldp4 | default('NONE')) != 'NONE' -%}
+            lldp4_fallback
+          {%- elif (dev2_passfile_used_lldp6 | default('NONE')) != 'NONE' -%}
+            lldp6_via_dev1
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable: tunnel auth={{ dev2_passfile_used_tunnel | default('n/a') }},
+          direct auth={{ dev2_passfile_used_direct | default('n/a') }},
+          lldp4 auth={{ dev2_passfile_used_lldp4 | default('n/a') }},
+          lldp6 auth={{ dev2_passfile_used_lldp6 | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "lldp4 passfile={{ dev2_passfile_used_lldp4 | default('n/a') }}"
+          - "lldp6 passfile={{ dev2_passfile_used_lldp6 | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} (forwarded to 127.0.0.1:{{ _local_port | default('na') }})"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for Phase 1 commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR: DEV2_CMD is empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp4_fallback)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_lldp4 }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp6_via_dev1)
+              IP6="{{ lldp_dev2_ip6 }}"
+              F="{{ dev2_passfile_used_lldp6 }}"
+              sshpass -p "{{ dev1_pass }}" ssh {{ ssh_opts_common }} \
+                "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+                "sshpass -f '/tmp/${F}' ssh \
+                   -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                   -o PubkeyAuthentication=no \
+                   -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                   -o ConnectTimeout={{ ssh_timeout }} \
+                   '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' \
+                   '{{ pathprefix }} '"${DEV2_CMD}" 2>&1"
+              ;;
+            *)
+              echo "ERROR: unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+
+    - name: Ensure destination directory on DEV2 (via wrapper)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='mkdir -p "{{ block_dst_dir }}"'
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      changed_when: false
+
+    - name: Copy block scripts controller to DEV2 direct (LLDP direct or LLDP4 fallback)
+      when: dev2_conn_final in ["direct_lldp", "lldp4_fallback"]
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        if [ "{{ dev2_conn_final }}" = "direct_lldp" ]; then
+          HOST="{{ lldp_dev2_ip }}"
+          PASSFILE="{{ dev2_passfile_used_direct }}"
+        else
+          HOST="{{ lldp_dev2_ip }}"
+          PASSFILE="{{ dev2_passfile_used_lldp4 }}"
+        fi
+        sshpass -f "$PASSFILE" scp {{ ssh_opts_common }} -P "{{ dev2_port }}" \
+          "{{ block_src_dir }}/{{ item }}" \
+          "{{ dev2_ssh_user }}@${HOST}:{{ block_dst_dir }}/{{ item }}"
+        sshpass -f "$PASSFILE" ssh {{ ssh_opts_common }} -p "{{ dev2_port }}" \
+          "{{ dev2_ssh_user }}@${HOST}" "chmod 0755 {{ block_dst_dir }}/{{ item }}"
+      args: { executable: /bin/bash }
+      loop: "{{ block_files }}"
+      changed_when: true
+
+    - name: Copy block scripts controller to DEV1 staging (for tunnel or LLDP6 via DEV1)
+      when: dev2_conn_final in ["tunnel", "lldp6_via_dev1"]
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p "{{ dev1_pass }}" scp {{ ssh_opts_common }} \
+          "{{ block_src_dir }}/{{ item }}" \
+          "{{ dev1_user }}@{{ inventory_hostname }}:/tmp/{{ item }}"
+      args: { executable: /bin/bash }
+      loop: "{{ block_files }}"
+      changed_when: true
+
+    - name: Stage DEV2 passfile on DEV1 for DEV1 to DEV2 scp
+      when: dev2_conn_final in ["tunnel", "lldp6_via_dev1"]
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PASSFILE="{{ dev2_passfile_used_tunnel if dev2_conn_final == 'tunnel' else dev2_passfile_used_lldp6 }}"
+        sshpass -p "{{ dev1_pass }}" scp {{ ssh_opts_common }} \
+          "$PASSFILE" \
+          "{{ dev1_user }}@{{ inventory_hostname }}:/tmp/ptsd_passfile_dev2"
+      args: { executable: /bin/bash }
+      changed_when: true
+
+    - name: Copy block scripts DEV1 to DEV2 (tunnel)
+      when: dev2_conn_final == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (PASSFILE="/tmp/ptsd_passfile_dev2";
+         for f in {{ block_files | join(' ') }}; do
+           sshpass -f "$PASSFILE" scp {{ ssh_opts_common }} -P {{ dev2_port }} "/tmp/$f" "{{ dev2_ssh_user }}@{{ dev2_host }}:{{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+           sshpass -f "$PASSFILE" ssh {{ ssh_opts_common }} -p {{ dev2_port }} "{{ dev2_ssh_user }}@{{ dev2_host }}" "chmod 0755 {{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+         done;
+         exit 0)
+      register: dev1_to_dev2_copy
+      changed_when: true
+      failed_when: dev1_to_dev2_copy.rc != 0
+
+    - name: Copy block scripts DEV1 to DEV2 (LLDP6 via DEV1)
+      when: dev2_conn_final == "lldp6_via_dev1"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (PASSFILE="/tmp/ptsd_passfile_dev2";
+         HOST6="{{ lldp_dev2_ip6 }}";
+         for f in {{ block_files | join(' ') }}; do
+           sshpass -f "$PASSFILE" scp {{ ssh_opts_common }} -6 -P {{ dev2_port }} "/tmp/$f" "{{ dev2_ssh_user }}@${HOST6}:{{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+           sshpass -f "$PASSFILE" ssh {{ ssh_opts_common }} -6 -p {{ dev2_port }} "{{ dev2_ssh_user }}@${HOST6}" "chmod 0755 {{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+         done;
+         exit 0)
+      register: dev1_to_dev2_copy6
+      changed_when: true
+      failed_when: dev1_to_dev2_copy6.rc != 0
+
+    - name: Make block scripts executable on DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='chmod 0755 /root/startblock.sh /root/stopblock.sh'
+        {{ dev2_exec_cmd }}
+      args:
+        executable: /bin/bash
+      changed_when: true      
+
+    - name: Publish redirect and block deployment journal
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': 'redirect and block script deployed'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Set NetBox custom field indoor_rnb -> deployed
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_rnb',
+            'task_result': 'deployed'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup: closing tunnel, removing temp IP, removing staged passfiles (best-effort)"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/redirect_and_block_remove.yml

@@ -0,0 +1,704 @@
+# redirect_and_block_removal.yml — connectivity + stage block scripts removal
+# Phase 0: reliable DEV2 connectivity + wrapper build
+# Phase 1: run twice and remove the scripts
+
+- name: "Redirect and block | Connectivity + stage scripts"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+    block_src_dir: "/opt/containers/ansible-worker/app"
+    block_dst_dir: "/root"
+    block_files:
+      - "startblock.sh"
+      - "stopblock.sh"
+
+  tasks:
+    - name: Initialize passfile facts (avoid undefined vars in later templates)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    # ============================ PHASE 0: LLDP DISCOVERY (IPv4 + IPv6) ============================
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    # ============================ PHASE 0: PRIMARY PATH DECISION ============================
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    # ============================ PHASE 0: DIRECT LLDP IPv4 AUTH (10.x) ============================
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ============================
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1 local_port to DEV2 port 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: SAFETY GUARD (MAC CHECK) ============================
+    - name: Read remote eth0 MAC via tunnel (guard, sanitized)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used_tunnel }}" ssh {{ ssh_opts_common }} \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /sys/class/net/eth0/address 2>/dev/null || ip link show eth0 2>/dev/null" 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_eth0_mac_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Normalize remote eth0 MAC (extract last MAC-like token)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        remote_eth0_mac: >-
+          {{
+            (
+              (dev2_eth0_mac_raw.stdout | default('') | regex_replace('\r','')) ~ "\n" ~
+              (dev2_eth0_mac_raw.stderr | default('') | regex_replace('\r',''))
+            )
+            | regex_findall('([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})')
+            | last
+            | default('')
+            | lower
+          }}
+      changed_when: false
+
+    - name: Abort if remote eth0 MAC != discovered DEV2 MAC
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+        - (remote_eth0_mac | default('') | length) > 0
+        - (remote_eth0_mac | lower) != (dev2_mac | lower)
+      ansible.builtin.fail:
+        msg: >-
+          Safety stop: tunnel reached wrong device.
+          expected_dev2_mac={{ dev2_mac }}, remote_eth0_mac={{ remote_eth0_mac }}
+
+    # ============================ PHASE 0: FALLBACKS (only if tunnel auth failed) ============================
+    - name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if timeout 20s sshpass -f "$f" ssh {{ ssh_opts_common }} \
+              -o AddressFamily=inet \
+              -o ConnectTimeout=10 \
+              "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp4
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv4 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp4: "{{ (dev2_auth_lldp4.rc == 0) | ternary((dev2_auth_lldp4.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    - name: Copy DEV2 passfiles to DEV1 for IPv6 nested SSH (last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      ansible.builtin.copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        mode: "0600"
+      loop: "{{ dev2_passfiles }}"
+      ignore_errors: true
+
+    - name: Try DEV2 login via LLDP IPv6 (nested SSH through DEV1; last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        IP6="{{ lldp_dev2_ip6 }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+              "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+              "timeout {{ ssh_timeout }}s sshpass -f '/tmp/${f}' ssh \
+                 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                 -o PubkeyAuthentication=no \
+                 -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                 -o ConnectTimeout={{ ssh_timeout }} \
+                 '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' 'echo OK' " \
+              >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp6
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv6 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp6: "{{ (dev2_auth_lldp6.rc == 0) | ternary((dev2_auth_lldp6.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    # ============================ PHASE 0: FINAL DECISION + WRAPPER FACTS ============================
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- elif (dev2_passfile_used_lldp4 | default('NONE')) != 'NONE' -%}
+            lldp4_fallback
+          {%- elif (dev2_passfile_used_lldp6 | default('NONE')) != 'NONE' -%}
+            lldp6_via_dev1
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable: tunnel auth={{ dev2_passfile_used_tunnel | default('n/a') }},
+          direct auth={{ dev2_passfile_used_direct | default('n/a') }},
+          lldp4 auth={{ dev2_passfile_used_lldp4 | default('n/a') }},
+          lldp6 auth={{ dev2_passfile_used_lldp6 | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "lldp4 passfile={{ dev2_passfile_used_lldp4 | default('n/a') }}"
+          - "lldp6 passfile={{ dev2_passfile_used_lldp6 | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} (forwarded to 127.0.0.1:{{ _local_port | default('na') }})"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for Phase 1 commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR: DEV2_CMD is empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp4_fallback)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_lldp4 }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp6_via_dev1)
+              IP6="{{ lldp_dev2_ip6 }}"
+              F="{{ dev2_passfile_used_lldp6 }}"
+              sshpass -p "{{ dev1_pass }}" ssh {{ ssh_opts_common }} \
+                "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+                "sshpass -f '/tmp/${F}' ssh \
+                   -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                   -o PubkeyAuthentication=no \
+                   -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                   -o ConnectTimeout={{ ssh_timeout }} \
+                   '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' \
+                   '{{ pathprefix }} '"${DEV2_CMD}" 2>&1"
+              ;;
+            *)
+              echo "ERROR: unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+
+    ##########################################################################
+    # PHASE 1: ROLLBACK (execute stop, remove scripts, journal + custom field)
+    ##########################################################################
+
+    - name: Run stopblock twice with 2s pause
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='/root/stopblock.sh >/dev/null 2>&1 || true; sleep 2; /root/stopblock.sh >/dev/null 2>&1 || true'
+        {{ dev2_exec_cmd }}
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Remove block scripts from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='rm -f /root/stopblock.sh /root/startblock.sh'
+        {{ dev2_exec_cmd }}
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Publish redirect and block removal journal
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': 'redirect and block removed'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Set NetBox custom field indoor_rnb to cleared
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_rnb',
+            'task_result': 'cleared'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup: closing tunnel, removing temp IP, removing staged passfiles (best-effort)"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/restart-cloud-agent-long-wrapper.yml

@@ -0,0 +1,44 @@
+# restart-cloud-agent-wrapper.yml
+# Phase 1: Subsystem -> NetBox sync for current inventory_hostname
+# Phase 2: Restart cloud-agent on the device
+
+- name: Phase 1 | Subsystem -> NetBox sync before cloud-agent restart
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: "Wrapper | Verify netbox_subsystem_ikejanum_recursive.py exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: ./netbox_subsystem_ikejanum_recursive.py
+      register: subsystem_script
+      changed_when: false
+
+    - name: "Wrapper | Run netbox_subsystem_ikejanum_recursive.py for {{ inventory_hostname }}"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -o pipefail
+        python3 -u ./netbox_subsystem_ikejanum_recursive.py "{{ inventory_hostname }}" 2>&1
+      args:
+        executable: /bin/bash
+      register: subsystem_sync
+      changed_when: false
+      failed_when: false
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Show subsystem sync result"
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "script_exists={{ subsystem_script.stat.exists | default(false) }}"
+          - "rc={{ subsystem_sync.rc | default('NA') }}"
+          - "stdout_lines={{ subsystem_sync.stdout_lines | default([]) }}"
+          - "stderr_lines={{ subsystem_sync.stderr_lines | default([]) }}"
+          - "raw stdout (joined): {{ subsystem_sync.stdout | default('') }}"
+          - "raw stderr (joined): {{ subsystem_sync.stderr | default('') }}"
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Abort softly if script missing"
+      ansible.builtin.meta: end_host
+      when: not (subsystem_script.stat.exists | default(false))
+
+- import_playbook: restart-cloud-agent-long.yml

files/ansible-playbooks/restart-cloud-agent-long.yml

@@ -0,0 +1,25 @@
+---
+- name: Temporarily move cloud-agent and restore it
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Move /tmp/launchd/services/cloud-agent to /root/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /tmp/launchd/services/cloud-agent /root/
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 200
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /root/cloud-agent /tmp/launchd/services/
+      register: move_back
+      changed_when: true

files/ansible-playbooks/restart-cloud-agent-wrapper.yml

@@ -0,0 +1,44 @@
+# restart-cloud-agent-wrapper.yml
+# Phase 1: Subsystem -> NetBox sync for current inventory_hostname
+# Phase 2: Restart cloud-agent on the device
+
+- name: Phase 1 | Subsystem -> NetBox sync before cloud-agent restart
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: "Wrapper | Verify netbox_subsystem_ikejanum_recursive.py exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: ./netbox_subsystem_ikejanum_recursive.py
+      register: subsystem_script
+      changed_when: false
+
+    - name: "Wrapper | Run netbox_subsystem_ikejanum_recursive.py for {{ inventory_hostname }}"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -o pipefail
+        python3 -u ./netbox_subsystem_ikejanum_recursive.py "{{ inventory_hostname }}" 2>&1
+      args:
+        executable: /bin/bash
+      register: subsystem_sync
+      changed_when: false
+      failed_when: false
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Show subsystem sync result"
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "script_exists={{ subsystem_script.stat.exists | default(false) }}"
+          - "rc={{ subsystem_sync.rc | default('NA') }}"
+          - "stdout_lines={{ subsystem_sync.stdout_lines | default([]) }}"
+          - "stderr_lines={{ subsystem_sync.stderr_lines | default([]) }}"
+          - "raw stdout (joined): {{ subsystem_sync.stdout | default('') }}"
+          - "raw stderr (joined): {{ subsystem_sync.stderr | default('') }}"
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Abort softly if script missing"
+      ansible.builtin.meta: end_host
+      when: not (subsystem_script.stat.exists | default(false))
+
+- import_playbook: restart-cloud-agent.yml

files/ansible-playbooks/restart-cloud-agent.yml

@@ -0,0 +1,25 @@
+---
+- name: Temporarily move cloud-agent and restore it
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Move /tmp/launchd/services/cloud-agent to /root/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /tmp/launchd/services/cloud-agent /root/
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 3
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /root/cloud-agent /tmp/launchd/services/
+      register: move_back
+      changed_when: true

files/ansible-playbooks/sot-updater-iponly.yml

@@ -0,0 +1,212 @@
+# sot-updater-iponly.yml — NetBox IP sync ONLY via nb_onedevice_update.py (no device scan)
+
+- name: NetBox IP sync only (Cloud → NetBox via nb_onedevice_update.py); do not scan devices
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Wrapper mode:
+    #   false (default): standalone behavior (may end_host)
+    #   true: composable behavior for sot-updater-wrapper.yml (no end_host)
+    sot_wrapper_mode: false
+
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=15
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }}"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Detect cloud failure"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      delegate_facts: true
+      vars:
+        _out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        _lines: >-
+          {{ _out.split('\n') | map('trim') | list }}
+        _cloud_fail_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
+        _error_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+') | list | last | default('')) }}
+        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
+        _cloud_error_type: >-
+          {%- if _error_line is search('network error')
+                and _error_line is search('500 error responses') -%}
+            cloud_http_500
+          {%- elif (_cloud_fail_line | length) > 0 -%}
+            cloud_logic
+          {%- elif _rc_is_3 -%}
+            cloud_other
+          {%- else -%}
+            none
+          {%- endif -%}
+        _cloud_retryable: >-
+          {{ _cloud_error_type not in ['cloud_http_500'] }}
+      set_fact:
+        cloud_fail_line: "{{ _cloud_fail_line }}"
+        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"
+        cloud_error_type: "{{ _cloud_error_type }}"
+        cloud_retryable: "{{ _cloud_retryable }}"
+
+    # --- Extract upgrade_cmd line if any ---
+    - name: "NB preflight | Extract upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\\s*upgrade_cmd\\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Parse upgrade_cmd value"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args: { executable: /bin/bash }
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd='{{ nb_upgrade_cmd }}' (len={{ nb_upgrade_cmd|length }})"
+
+    - name: "NB preflight | Parse OK line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      delegate_facts: true
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=') 
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc|default(1)) == 0 and (nb_ok_line|length)>0 }}"
+        nb_dev: "{{ (nb_tokens|first|default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip','') }}"
+        nb_fw: "{{ nb_kv.get('fw','') }}"
+        nb_node: "{{ nb_kv.get('node','') }}"
+        nb_sector: "{{ nb_kv.get('sector','') }}"
+        nb_small: "{{ nb_kv.get('small','') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    - name: "NB preflight | Detect IP change"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      delegate_facts: true
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      debug:
+        msg:
+          - "Preflight verdict:"
+          - "Cloud fail: {{ cloud_bad|default(false) }}"
+          - "IP changed: {{ nb_ip_changed|default(false) }}"
+          - "Reason: {{ nb_change_reason|default('none') }}"
+
+    - name: "NB preflight | Pause 1s if OK"
+      when:
+        - nb_script.stat.exists
+        - not ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
+      delegate_to: localhost
+      pause:
+        seconds: 1
+
+  tasks:
+    - name: "IP-only | Stop after NetBox sync (no device scan)"
+      when: not (sot_wrapper_mode | default(false))
+      meta: end_host

files/ansible-playbooks/sot-updater-scheduler.yml

@@ -0,0 +1,194 @@
+# sot-updater-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run full sot-updater-current.yml
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('(?:^|\\s)ip=([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)(?:\\s|$)', '\\1') | default('') }}"
+        _ip: >-
+          {%- if _ip_raw is string -%}
+            {{ _ip_raw }}
+          {%- elif _ip_raw is sequence and (_ip_raw | length) > 0 -%}
+            {{ _ip_raw[0] }}
+          {%- else -%}
+            ""
+          {%- endif -%}
+      set_fact:
+        wrapper_ok_line: "{{ _ok_line }}"
+        wrapper_fail_line: "{{ _fail_line }}"
+        wrapper_nb_ip: "{{ _ip | trim }}"
+
+
+    - name: "Wrapper | Debug extracted values"
+      ansible.builtin.debug:
+        msg:
+          - "wrapper_ok_line='{{ wrapper_ok_line }}'"
+          - "wrapper_fail_line='{{ wrapper_fail_line | default('') }}'"
+          - "wrapper_nb_ip='{{ wrapper_nb_ip }}'"
+
+    # ----------------------- Option A: Cloud offline / no IP -> Journal + stop cleanly -----------------------
+    - name: "Wrapper | Journal: device not online in cloud (skip full scan)"
+      delegate_to: localhost
+      when: (wrapper_fail_line | default('') | length) > 0
+      vars:
+        _journal_msg: >-
+          during sot-updater scan the device was not online in cloud (no IP). please try later.
+          details: {{ wrapper_fail_line | default('') }}
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': _journal_msg
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: "Wrapper | Mark host to skip full scan (cloud-offline)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      set_fact:
+        wrapper_skip_full_scan: true
+
+
+    - name: "Wrapper | Stop host after cloud-offline journal (no full scan)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      meta: end_host
+
+    # ----------------------- Normal path: we have OK ip=... -> proceed -----------------------
+    - name: "Wrapper | Assert IP extracted successfully"
+      ansible.builtin.assert:
+        that:
+          - nb_script is defined
+          - nb_script.stat.exists | default(false)
+          - (wrapper_ok_line | length) > 0
+          - (wrapper_nb_ip | length) > 0
+        fail_msg: >-
+          Wrapper stop: could not extract IP from nb_onedevice_update.py output.
+          ok_line='{{ wrapper_ok_line | default('') }}'
+          fail_line='{{ wrapper_fail_line | default('') }}'
+          stdout_tail={{ (nb_preflight.stdout_lines | default([]))[-12:] }}
+
+    - name: "Wrapper | Override ansible_host to refreshed IP ({{ wrapper_nb_ip }})"
+      set_fact:
+        ansible_host: "{{ wrapper_nb_ip }}"
+
+    - name: "Wrapper | Reset SSH connection context (ensure new ansible_host is used)"
+      meta: reset_connection
+
+- name: Phase 3 gate | Skip full scan if cloud-offline
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: "Wrapper | Gate: end_host if wrapper_skip_full_scan is set"
+      when: wrapper_skip_full_scan | default(false)
+      meta: end_host
+
+
+- import_playbook: sot-updater.yml
+
+
+- name: Phase 4 | Update action fields after sot-updater
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Scheduler | Set action_state done"
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'done'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: "Scheduler | Set action_last sot-updater-scheduler"
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_last',
+            'task_result': 'sot-updater-scheduler'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/sot-updater-upgradecmd.yml

@@ -0,0 +1,752 @@
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # === NEW (surgical): requeue attempt counter from environment, default 0 ===
+    requeue_attempt: "{{ (lookup('env','REQUEUE_ATTEMPT') | default('0', true)) | int }}"
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing (path typo?)"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }} (chatty)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    # === HARD STOP ON CLOUD FAILURE (immediately after preflight) ===
+    - name: "NB preflight | Detect cloud failure (rc==3 OR 'FAIL Cloud' line)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        _lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r',''))
+             | split('\n') | map('trim') | list }}
+        _cloud_fail_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
+        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
+      ansible.builtin.set_fact:
+        cloud_fail_line: "{{ _cloud_fail_line }}"
+        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"
+
+    - name: "NB preflight | Journal + STOP (cloud unavailable)"
+      when:
+        - nb_script.stat.exists
+        - cloud_bad | default(false)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'preflight abort: cloud unavailable; closing without requeue. '
+              ~ (cloud_fail_line | default(''))
+              ~ ' rc=' ~ ((nb_preflight.rc | default('')) | string)
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cloud_unavail
+      changed_when: (rmq_cloud_unavail.json is defined) and (rmq_cloud_unavail.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: "NB preflight | Stop host due to cloud unavailability (no requeue)"
+      when:
+        - nb_script.stat.exists
+        - cloud_bad | default(false)
+      ansible.builtin.meta: end_host
+    # === END HARD STOP ===
+
+    # --- Debug-only: capture the exact upgrade_cmd line and stop the play ---
+    - name: "NB preflight | Extract exact upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\s*upgrade_cmd\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Show captured upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "upgrade_cmd line (exact): {{ nb_upgrade_line | default('<not found>') }}"
+          - "found? {{ (nb_upgrade_line | length) > 0 }}"
+
+    - name: "NB preflight | Parse value after '=' via awk"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args:
+        executable: /bin/bash
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set and show parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd value: '{{ nb_upgrade_cmd }}'"
+          - "len={{ nb_upgrade_cmd | length }}"
+
+    # --- Parse the OK line robustly (token-based) ---
+    - name: "NB preflight | Parse OK line (token)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=')
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc | default(1)) == 0 and (nb_ok_line | length) > 0 }}"
+        nb_dev: "{{ (nb_tokens | first | default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip', '') }}"
+        nb_fw: "{{ nb_kv.get('fw', '') }}"
+        nb_node: "{{ nb_kv.get('node', '') }}"
+        nb_sector: "{{ nb_kv.get('sector', '') }}"
+        nb_small: "{{ nb_kv.get('small', '') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    # --- Detect "cloud vs NetBox (before update) was different" (regex-free, robust)
+    - name: "NB preflight | Detect whether IP changed (pre-update)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight verdict: {{ 'IP CHANGED (will requeue)' if nb_ip_changed else 'IP SAME (continue)' }}"
+          - "Reason: {{ nb_change_reason }}"
+
+    # --- If IP changed → publish a 3s delayed 'sot-updater' and stop this host ---
+    - name: "NB preflight | Publish delayed requeue (3s) and stop"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+        - (requeue_attempt | int) < 3              # <— NEW: limit to 3 tries
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            # === NEW (surgical): 20s delay instead of 3s ===
+            headers: { x-delay: 20000 }
+          routing_key: "deviceconfig"
+          # === NEW (surgical): include attempt counter in payload ===
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'sot-updater',
+            'attempt': ((requeue_attempt | int) + 1)
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_requeue
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Log requeue publish response"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+        - (requeue_attempt | int) < 3              # <— NEW: only if we actually published
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "RMQ publish URL: http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+          - "HTTP status: {{ rmq_requeue.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_requeue.json | default('<none>') }}"
+          - "Raw content: {{ rmq_requeue.content | default('<none>') }}"
+
+    - name: "NB preflight | Stop further tasks for this host"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+      meta: end_host
+
+    # --- If IP did not change → optional 1s pause, then continue normally ---
+    - name: "NB preflight | Pause 1s"
+      when:
+        - nb_script.stat.exists
+        - not (nb_ip_changed | default(false))
+      delegate_to: localhost
+      ansible.builtin.pause:
+        seconds: 1
+
+  tasks:
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug result of adding temp IP to DEV1
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_mac is defined and dev2_mac | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug ARP action summary on DEV1
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- ARP refresh ----------------------------
+    - name: Refresh ARP #1
+      ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Refresh ARP #1
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Local tunnel prep -----------------------
+    - name: Pick a free local TCP port for the tunnel
+      delegate_to: localhost
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Record chosen port and create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set facts for tunnel paths
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
+
+    # ----------------------- AIRPING #2 -----------------------
+    - name: Refresh ARP #2
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Start tunnel -----------------------
+    - name: Start SSH tunnel via DEV1
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- DEV1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
+
+    - name: Debug Dev1 normalized firmware
+      delegate_to: localhost
+      debug:
+        msg: "Dev1 fw_version  {{ dev1_fw_clean | default('N/A') }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- DEV2 auth -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok | default(false)
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set dev2_passfile_used
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
+
+    # ----------------------- AIRPING #3 -----------------------
+    - name: Refresh ARP #3
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- DEV2 firmux (simplified, literal) -----------------------
+    - name: Dev2 | Read /usr/lib/release/firmux
+      when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
+      delegate_to: localhost
+      shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      register: dev2_firmux
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
+      delegate_to: localhost
+      run_once: true
+      set_fact:
+        indoor_fw_norm: >-
+          {{
+            (
+              (
+                dev2_firmux.stdout | default('') | regex_replace('\r','')
+              ) | split('\n')
+              | map('trim')
+              | select('truthy')
+              | list
+            ) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
+              | list
+              | first
+              | default(
+                  ((dev2_firmux.stdout | default('') | regex_replace('\r',''))
+                  | split('\n')
+                  | map('trim')
+                  | select('truthy')
+                  | list
+                  | first
+                  | default(''))
+                )
+              | trim
+          }}
+
+    - name: Publish Dev2 indoor_fwver
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP #4
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster
+      delegate_to: localhost
+      shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir
+      delegate_to: localhost
+      file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    # --- Determine success of Dev1 & Dev2 reads (controller-side heuristics) ---
+    - name: Post | Derive success flags for Dev1/Dev2 reads
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev1_read_ok: "{{ (dev1_fw_clean | default('') | trim) | length > 0 }}"
+        dev2_read_ok: "{{ (dev2_firmux is defined) and ((dev2_firmux.rc | default(1)) == 0) and ((dev2_firmux.stdout | default('') | trim) | length > 0) }}"
+
+    - name: Post | Debug success flags
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev1_read_ok={{ dev1_read_ok | default(false) }}"
+          - "dev2_read_ok={{ dev2_read_ok | default(false) }}"
+          - "nb_upgrade_cmd='{{ (nb_upgrade_cmd | default('')) }}' (len={{ (nb_upgrade_cmd | default('')) | length }})"
+
+    # --- If both reads OK and upgrade_cmd looks valid, publish journal + schedule upgrade ---
+    - name: Post | Build journal payload for planned upgrade
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        upgrade_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            device is configured to be upgraded with {{ nb_upgrade_cmd }}. requesting the upgrade
+
+    - name: Post | Publish journal to controls
+      when: upgrade_journal_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ upgrade_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_upgrade
+      changed_when: (rmq_journal_upgrade.json is defined) and (rmq_journal_upgrade.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: Post | Schedule upgrade task via delayed exchange (10s)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers: { x-delay: 300000 }
+          routing_key: "deviceconfig"
+          payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': (nb_upgrade_cmd | default('')) } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_schedule_upgrade
+      changed_when: false
+      failed_when: false
+
+    # === NEW: Tag only when scheduling path ran (success gate same as scheduling) ===
+    - name: Post | Build tag payload (auto-indoor-upgrade)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+        - rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_add1: "auto-indoor-upgrade"
+
+    - name: Post | Publish add-tag to controls
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add
+      changed_when: (rmq_tag_add.json is defined) and (rmq_tag_add.json.routed | default(false) | bool)
+
+    - name: Post | Log upgrade scheduling response
+      when: rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "Upgrade scheduled to deviceconfig.delayed in 10s"
+          - "HTTP status: {{ rmq_schedule_upgrade.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_schedule_upgrade.json | default('<none>') }}"
+          - "Raw content: {{ rmq_schedule_upgrade.content | default('<none>') }}"
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE"

files/ansible-playbooks/sot-updater-wrapper.yml

@@ -0,0 +1,134 @@
+# sot-updater-wrapper.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run full sot-updater-current.yml
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('(?:^|\\s)ip=([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)(?:\\s|$)', '\\1') | default('') }}"
+        _ip: >-
+          {%- if _ip_raw is string -%}
+            {{ _ip_raw }}
+          {%- elif _ip_raw is sequence and (_ip_raw | length) > 0 -%}
+            {{ _ip_raw[0] }}
+          {%- else -%}
+            ""
+          {%- endif -%}
+      set_fact:
+        wrapper_ok_line: "{{ _ok_line }}"
+        wrapper_fail_line: "{{ _fail_line }}"
+        wrapper_nb_ip: "{{ _ip | trim }}"
+
+
+    - name: "Wrapper | Debug extracted values"
+      ansible.builtin.debug:
+        msg:
+          - "wrapper_ok_line='{{ wrapper_ok_line }}'"
+          - "wrapper_fail_line='{{ wrapper_fail_line | default('') }}'"
+          - "wrapper_nb_ip='{{ wrapper_nb_ip }}'"
+
+    # ----------------------- Option A: Cloud offline / no IP -> Journal + stop cleanly -----------------------
+    - name: "Wrapper | Journal: device not online in cloud (skip full scan)"
+      delegate_to: localhost
+      when: (wrapper_fail_line | default('') | length) > 0
+      vars:
+        _journal_msg: >-
+          during sot-updater scan the device was not online in cloud (no IP). please try later.
+          details: {{ wrapper_fail_line | default('') }}
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': _journal_msg
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: "Wrapper | Mark host to skip full scan (cloud-offline)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      set_fact:
+        wrapper_skip_full_scan: true
+
+
+    - name: "Wrapper | Stop host after cloud-offline journal (no full scan)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      meta: end_host
+
+    # ----------------------- Normal path: we have OK ip=... -> proceed -----------------------
+    - name: "Wrapper | Assert IP extracted successfully"
+      ansible.builtin.assert:
+        that:
+          - nb_script is defined
+          - nb_script.stat.exists | default(false)
+          - (wrapper_ok_line | length) > 0
+          - (wrapper_nb_ip | length) > 0
+        fail_msg: >-
+          Wrapper stop: could not extract IP from nb_onedevice_update.py output.
+          ok_line='{{ wrapper_ok_line | default('') }}'
+          fail_line='{{ wrapper_fail_line | default('') }}'
+          stdout_tail={{ (nb_preflight.stdout_lines | default([]))[-12:] }}
+
+    - name: "Wrapper | Override ansible_host to refreshed IP ({{ wrapper_nb_ip }})"
+      set_fact:
+        ansible_host: "{{ wrapper_nb_ip }}"
+
+    - name: "Wrapper | Reset SSH connection context (ensure new ansible_host is used)"
+      meta: reset_connection
+
+- name: Phase 3 gate | Skip full scan if cloud-offline
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: "Wrapper | Gate: end_host if wrapper_skip_full_scan is set"
+      when: wrapper_skip_full_scan | default(false)
+      meta: end_host
+
+
+- import_playbook: sot-updater.yml

files/ansible-playbooks/update-indoor-222r6820-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor222-r6820.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-223r6828-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor223-r6828.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-224r6820-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor222-r6820.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-224r6831-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor224-r6831.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-225r6848-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor225-r6848.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-old.yml

@@ -1,4 +1,3 @@
----
 # update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
@@ -70,6 +69,28 @@
           - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
           - "inventory_hostname={{ inventory_hostname }}"
 
+    # ====== NEW: pick up force-upgrade flag from CLI and TEMP override ======
+    - name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes)
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | trim }}"
+
+    # >>> TEMPORARY OVERRIDE (REMOVE THIS TASK LATER) <<<
+    - name: TEMPORARILY set force-upgrade to yes (REMOVE BEFORE COMMITTING)
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "no"
+    # <<< END TEMPORARY >>>
+
+    - name: Normalize force-upgrade flag to boolean
+      ansible.builtin.set_fact:
+        force_upgrade: "{{ (force_upgrade_raw | string | trim) | bool }}"
+
+    - name: Debug force-upgrade effective
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "force_upgrade_raw={{ force_upgrade_raw | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+
     - name: Stop early if connected DEV1 hostname != inventory
       ansible.builtin.meta: end_host
       when: (dev1_host_read.stdout | trim | length > 0) and
@@ -618,6 +639,126 @@
       ansible.builtin.debug:
         msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
 
+    # ====== INSERTED: two-step normalization preview (rev->r, then space->dash) ======
+    - name: "Normalize DEV2 firmux (step 1): replace 'rev ' -> 'r' (debug only)"
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _firmux_step1: "{{ (dev2_firmux.stdout | default('') | trim) | regex_replace('(?i)rev\\s+','r') }}"
+      changed_when: false
+
+    - name: "Debug firmux after step 1 (rev->r)"
+      when: _firmux_step1 is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "_firmux_step1={{ _firmux_step1 }}"
+
+    - name: "Normalize DEV2 firmux (step 2): replace space -> '-' (debug only)"
+      when: _firmux_step1 is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _firmux_step2: "{{ _firmux_step1 | replace(' ', '-') }}"
+      changed_when: false
+
+    - name: "Debug firmux after step 2 (space->dash)"
+      when: _firmux_step2 is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "_firmux_step2={{ _firmux_step2 }}"
+    # ====== END INSERTED ======
+
+
+    # ===================== Early equality/substring check (same logic as checker) =====================
+    # 1) expected_norm from image_filename (prefer extracted X.Y.Z-rNNNN)
+    - name: Normalize expected target step one compute base string (from image_filename)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_step1: "{{ (image_filename | default('') | trim) }}"
+      changed_when: false
+
+    - name: Extract version core X dot Y dot Z dash rNNNN from image_filename if present
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core_list: "{{ (image_filename | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
+      changed_when: false
+
+    - name: Choose first extracted core if available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
+      changed_when: false
+
+    - name: Normalize expected target step two prefer extracted core when available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+      changed_when: false
+
+    # 2) banner_raw + banner_norm (convert "rev NNNN" → "-rNNNN" only if needed)
+    - name: Capture firmware banner line from DEV2 (raw)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_raw: "{{ (dev2_firmux.stdout | default('') | trim) }}"
+      changed_when: false
+
+    - name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_norm: >-
+          {{
+            ((banner_raw | lower) is search('-r[0-9]+$'))
+            | ternary(
+                banner_raw,
+                (banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1'))
+              )
+          }}
+      changed_when: false
+
+    # 3) evaluate (same equality OR substring, case-insensitive)
+    - name: Evaluate version match using normalized equality or substring
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        version_match: >-
+          {{
+            (expected_norm | default('') | length > 0)
+            and (
+              (banner_norm | default('')) == (expected_norm | default(''))
+              or ((banner_norm | default('') | lower) is search((expected_norm | default('') | lower)))
+              or ((expected_norm | default('') | lower) is search((banner_norm | default('') | lower)))
+            )
+          }}
+      changed_when: false
+
+    - name: Debug  version compare snapshot (pre-write)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm={{ expected_norm | default('') }}"
+          - "banner_raw={{ banner_raw | default('') }}"
+          - "banner_norm={{ banner_norm | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+          - "version_match={{ version_match | default(false) }}"
+
+    # 4) soft-block when equal (unless forced) + back-fill legacy debug names
+    - name: Soft-block if already on target (skip staging/write unless force-upgrade)
+      when:
+        - not (force_upgrade | default(false) | bool)
+        - version_match | default(false)
+      ansible.builtin.set_fact:
+        _blocked: true
+        _journal: "{{ (_journal | default([])) + [ 'Device already running target image: ' ~ (banner_norm | default('')) ~ ' — skipping write/flip (use force-upgrade=yes to override)' ] }}"
+        expected_fw_core_early: "{{ expected_norm | default('') }}"
+        current_fw_core: "{{ banner_norm | default('') }}"
+      changed_when: false
+
+    # If we are NOT blocked, still fill the legacy names so your later summary prints them
+    - name: Back-fill compare names for summary (no-op if already set)
+      when: not (_blocked | default(false))
+      ansible.builtin.set_fact:
+        expected_fw_core_early: "{{ expected_fw_core_early | default(expected_norm | default('')) }}"
+        current_fw_core: "{{ current_fw_core | default(banner_norm | default('')) }}"
+      changed_when: false
+
     # ---------------------------- Normalize rebootin early (HOURS) ----------------------------
     # Strict: rebootin is integer HOURS only. Always add +20s grace to the schedule.
 
@@ -952,7 +1093,7 @@
       block:
 
         # --- supervised async write with stall handling ---
-        - name: Write image to inactive bank on DEV2 (update -w, supervised async)
+        - name: Launch update -w on DEV2 (async)
           delegate_to: localhost
           vars:
             _write_async_cap: 600     # 10 minutes max runtime (tune if needed)
@@ -1327,13 +1468,16 @@
       when: afterupgrade_payload is defined
       delegate_to: localhost
 
-    # ---------------------------- Final operator summary (concise) ----------------------------
+    # ---------------------------- Final operator summary (one-liners) ----------------------------
     - name: Summary key outcomes (one-liners)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
           - "dev2_passfile_used={{ dev2_passfile_used }}"
           - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+          - "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
+          - "current_fw_core={{ current_fw_core | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
           - "local_image_present={{ local_img.stat.exists | default(false) }}"
           - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
           - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"

files/ansible-playbooks/update-indoor-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor6-stable.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoorbackup223r6828-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoorbackup223r6828.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoorbackup224r6831-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoorbackup224r6831.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoorbackup225r6848-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoorbackup225r6848.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-outdoor-bootenv.yml

@@ -0,0 +1,791 @@
+- name: Fix outdoor bootenv safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # Future post-bootenv check scheduling (kept disabled for now)
+    afterbootenv_hold_exchange: "{{ lookup('env','AFTERBOOTENV_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterbootenv_routing_key: "{{ lookup('env','AFTERBOOTENV_ROUTING_KEY') | default('deviceconfig', true) }}"
+    afterbootenv_hold_queue: "{{ lookup('env','AFTERBOOTENV_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+    afterbootenv_check_enabled: false
+
+    bootenv_filename: "fox100_bootenv.bin"
+    bootenv_sha256: "324337e20b0a2d8048c359bfa2a1b8dffd6b1a28eab260143dd895ca39c034aa"
+    bootenv_path: "/tmp/{{ bootenv_filename }}"
+    bootenv_mtd_device: "/dev/mtdblock8"
+    bootenv_expected_size: "65536"
+
+    # Helper computed vars
+    bootenv_name: "{{ bootenv_filename | regex_replace('\\.bin$', '') }}"
+    bootenv_marker: "/tmp/bootenv_fixed_{{ bootenv_name }}"
+    bootenv_lock_marker: "/tmp/bootenv_fix_inprogress_{{ bootenv_name }}"
+    firmware_guard_marker: "/tmp/prepared_for_{{ bootenv_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping bootenv steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Bootenv fix cancelled because a prepared marker is already present; " ~
+                  "expected marker " ~ firmware_guard_marker ~ ". Skipping bootenv steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    - name: Check if bootenv lock marker already exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e '{{ bootenv_lock_marker }}' ] && echo PRESENT || echo ABSENT"
+      register: bootenv_lock_scan
+      changed_when: false
+
+    - name: Debug bootenv lock marker presence
+      ansible.builtin.debug:
+        msg: "bootenv_lock_marker={{ bootenv_lock_scan.stdout | trim }}"
+
+    - name: Journal bootenv fix already in progress, skipping
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Bootenv fix cancelled because lock marker already exists; " ~
+                  "marker " ~ bootenv_lock_marker ~ ". Another process may be running."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_lock_present
+      changed_when: (rmq_journal_lock_present.json is defined) and (rmq_journal_lock_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_lock_present.status != 200) or
+        (rmq_journal_lock_present.json is not defined) or
+        (not (rmq_journal_lock_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
+
+    - name: End play for this host (bootenv lock already present)
+      ansible.builtin.meta: end_host
+      when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
+
+    - name: Create bootenv lock marker
+      ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_lock_marker }}'"
+      changed_when: true
+
+    - name: Bootenv fix main block
+      block:
+        - name: Count fw_printenv lines before bootenv write
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count_before
+          changed_when: false
+
+        - name: Debug fw_printenv line count before bootenv write
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines_before={{ env_line_count_before.stdout | trim }}"
+
+        - name: Note if bootloader environment looks invalid before bootenv write (<7 lines)
+          ansible.builtin.debug:
+            msg: "Proceeding with bootenv repair even though fw_printenv returned only {{ env_line_count_before.stdout | trim }} lines (<7) before bootenv write."
+          when: (env_line_count_before.stdout | trim | int) < 7
+
+        - name: Check if bootenv image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ bootenv_path }}' ] && echo OK || echo MISSING"
+          register: bootenv_exists
+          changed_when: false
+
+        - name: Upload bootenv to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ bootenv_src_local | default("/opt/containers/ansible-worker/files/fox100_bootenv.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ bootenv_path }}"
+          delegate_to: localhost
+          when: bootenv_exists.stdout is not defined or (bootenv_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check bootenv presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ bootenv_path }}' && echo OK || echo MISSING"
+          register: bootenv_exists2
+          changed_when: false
+          failed_when: (bootenv_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded bootenv image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ bootenv_path }}' | awk '{print $1}'"
+          register: bootenv_sha_out
+          changed_when: false
+
+        - name: Verify bootenv sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ bootenv_path }}. Got {{ bootenv_sha_out.stdout | trim }}, expected {{ bootenv_sha256 }}"
+          when: (bootenv_sha_out.stdout | trim) != (bootenv_sha256 | trim)
+
+        - name: Bootenv sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ bootenv_sha_out.stdout | trim }}"
+
+        - name: Read bootenv image size on target
+          ansible.builtin.raw: "{{ pathprefix }} wc -c < '{{ bootenv_path }}'"
+          register: bootenv_size_out
+          changed_when: false
+
+        - name: Verify bootenv image size matches expected
+          ansible.builtin.fail:
+            msg: "Bootenv size mismatch for {{ bootenv_path }}. Got {{ bootenv_size_out.stdout | trim }}, expected {{ bootenv_expected_size }}"
+          when: (bootenv_size_out.stdout | trim) != (bootenv_expected_size | string | trim)
+
+        - name: Bootenv size verification debug
+          ansible.builtin.debug:
+            msg:
+              - "bootenv_size={{ bootenv_size_out.stdout | trim }}"
+
+        - name: Write bootenv to flash with dd
+          ansible.builtin.raw: "{{ pathprefix }} dd if='{{ bootenv_path }}' of={{ bootenv_mtd_device }}"
+          register: bootenv_dd
+          changed_when: true
+          failed_when: bootenv_dd.rc != 0
+
+        - name: Debug dd output (bootenv)
+          ansible.builtin.debug:
+            msg:
+              - "dd.rc={{ bootenv_dd.rc | default('NA') }}"
+              - "dd.stdout={{ (bootenv_dd.stdout | default('')) | trim }}"
+              - "dd.stderr={{ (bootenv_dd.stderr | default('')) | trim }}"
+
+        - name: Run sync after bootenv write
+          ansible.builtin.raw: "{{ pathprefix }} sync"
+          register: bootenv_sync
+          changed_when: true
+          failed_when: bootenv_sync.rc != 0
+
+        - name: Debug sync output (bootenv)
+          ansible.builtin.debug:
+            msg:
+              - "sync.rc={{ bootenv_sync.rc | default('NA') }}"
+              - "sync.stdout={{ (bootenv_sync.stdout | default('')) | trim }}"
+              - "sync.stderr={{ (bootenv_sync.stderr | default('')) | trim }}"
+
+        - name: Set bootenv write success flag
+          ansible.builtin.set_fact:
+            _bootenv_write_success: "{{ (bootenv_dd.rc | default(1)) == 0 and (bootenv_sync.rc | default(1)) == 0 }}"
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ firmware_guard_marker }}'"
+          changed_when: true
+
+        - name: Create bootenv marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target bootenv)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "bootenv_outdoor_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ bootenv_filename }}"
+          when: _bootenv_write_success | bool
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Bootenv target file: {{ bootenv_filename }}"
+              - "Bootenv SHA256: OK ({{ bootenv_sha_out.stdout | trim }})"
+              - "Bootenv size: {{ bootenv_size_out.stdout | trim }} bytes"
+              - "fw_printenv lines before write: {{ env_line_count_before.stdout | trim }}"
+              - "dd: OK"
+              - "sync: OK"
+              - "Marker: {{ bootenv_marker }}"
+              - "Guard marker: {{ firmware_guard_marker }}"
+
+        - name: Compute reboot delay in seconds (default immediate when rebootin missing)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | default(0) | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Bootenv fix complete using {{ bootenv_filename }} written to {{ bootenv_mtd_device }}.
+                Original fw_printenv line count was {{ env_line_count_before.stdout | trim }}.
+                SHA256 OK {{ bootenv_sha_out.stdout | trim }}.
+                Size {{ bootenv_size_out.stdout | trim }} bytes.
+                Marker {{ bootenv_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate bootenv change.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate bootenv change.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Init after-bootenv scheduling vars
+          ansible.builtin.set_fact:
+            ab_attempt: 1
+            ab_max_attempts: 3
+            ab_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-bootenv check payload (attempt 1)
+          ansible.builtin.set_fact:
+            ab_attempt: 1
+            ab_delay_sec: "{{ ab_delay_sec | default(300) }}"
+            ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterbootenv_payload:
+              task_name: "afterbootenv_outdoor_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_bootenv_file: "{{ bootenv_filename }}"
+              original_fw_printenv_lines: "{{ env_line_count_before.stdout | trim }}"
+              attempt: "{{ ab_attempt | default(1) }}"
+              max_attempts: "{{ ab_max_attempts | default(3) }}"
+              current_delay_sec: "{{ ab_delay_sec | default(300) }}"
+              correlation_id: "{{ ab_correlation_id }}"
+              original_emitted_at: "{{ ab_original_emitted_at }}"
+              schema_version: 1
+              is_run_by: "{{ is_run_by_effective }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (ab_delay_sec | int) * 1000 }}"
+          when: afterbootenv_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-bootenv payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterbootenv_payload={{ afterbootenv_payload | to_nice_json }}"
+              - "ab_delay_sec={{ ab_delay_sec }}"
+              - "afterbootenv_check_enabled={{ afterbootenv_check_enabled }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+        - name: Bandaid | Pause before Publish delayed after-bootenv check to holding exchange
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when:
+            - afterbootenv_payload is defined
+            - afterbootenv_check_enabled | bool
+
+        - name: Publish delayed after-bootenv check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (ab_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterbootenv_routing_key }}"
+              payload: "{{ afterbootenv_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterbootenv_resp
+          changed_when: (rmq_afterbootenv_resp.json is defined) and (rmq_afterbootenv_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterbootenv_resp.status != 200) or
+            (rmq_afterbootenv_resp.json is not defined)
+          when:
+            - afterbootenv_payload is defined
+            - afterbootenv_check_enabled | bool
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state waiting (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'waiting'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Compute action_restart_timestamp (unix seconds) when rebootin == 0
+          ansible.builtin.set_fact:
+            action_restart_timestamp: "{{ lookup('pipe','date -u +%s') | int }}"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - reboot_seconds is defined
+            - (reboot_seconds | int) == 0
+
+        - name: Scheduler | Compute action_restart_timestamp (unix seconds) when reboot scheduled
+          ansible.builtin.set_fact:
+            action_restart_timestamp: "{{ (lookup('pipe','date -u +%s') | int) + (reboot_seconds | int) }}"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - reboot_seconds is defined
+            - (reboot_seconds | int) > 0
+
+        - name: Scheduler | Debug computed action_restart_timestamp
+          ansible.builtin.debug:
+            msg: "action_restart_timestamp={{ action_restart_timestamp }} (reboot_seconds={{ reboot_seconds | int }})"
+          when:
+            - is_run_by_effective == 'scheduler'
+            - action_restart_timestamp is defined
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_restart_timestamp custom field
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_restart_timestamp',
+                'task_result': (action_restart_timestamp | string)
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - action_restart_timestamp is defined
+
+        - name: Remove bootenv lock marker after success
+          ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
+          changed_when: true
+
+      rescue:
+        - name: Remove bootenv lock marker after failure
+          ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
+          changed_when: true
+          ignore_errors: true
+
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (bootenv failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (bootenv failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (bootenv failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-outdoorbackup-scheduler.yml

@@ -0,0 +1,156 @@
+# update-outdoorbackup-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-outdoorbackup224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+      
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-outdoorbackup225.yml
+  vars:
+    is_run_by: "scheduler"
+
+- name: Phase 4 | Scheduled success only | Chain sot-updater-scheduler
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks:
+    - name: Scheduled success | Set action_next to sot-updater-scheduler
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success | Publish sot-updater-scheduler work message
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/update-outdoorbackup224.yml

@@ -0,0 +1,800 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.4-r9850.bin
+    firmware_sha256: "38f7dd3bb5b06a2267d7bc68e2d8351df59c2aea858d644909208e11a3970539"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read /var/run/bootbanks.json (active and backup firmux versions)
+          ansible.builtin.raw: "{{ pathprefix }} cat /var/run/bootbanks.json"
+          register: bootbanks_raw
+          changed_when: false
+
+        - name: Parse bootbanks.json
+          ansible.builtin.set_fact:
+            bootbanks_obj: "{{ bootbanks_raw.stdout | from_json }}"
+            bootbanks_active_firmux: "{{ (bootbanks_raw.stdout | from_json).active.firmux | default('unknown') }}"
+            bootbanks_backup_firmux: "{{ (bootbanks_raw.stdout | from_json).backup.firmux | default('unknown') }}"
+
+        - name: current versions (active primary and backup)
+          ansible.builtin.debug:
+            msg:
+              - "active (primary) firmux: {{ bootbanks_active_firmux }}"
+              - "backup firmux: {{ bootbanks_backup_firmux }}"
+
+        - name: Build control queue payload for skip journal (backup already prepared)
+          ansible.builtin.set_fact:
+            journal_skip_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bootbank already has {{ fw_banner_repr }} (backup.firmux={{ bootbanks_backup_firmux }}).
+                Skipping backup preparation (no flip, no reboot).
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+        - name: Bandaid | Pause before Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when: journal_skip_payload is defined
+
+        - name: Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_skip_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_skip_resp
+          changed_when: (rmq_journal_skip_resp.json is defined) and (rmq_journal_skip_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_skip_resp.status != 200) or
+            (rmq_journal_skip_resp.json is not defined) or
+            (not (rmq_journal_skip_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: journal_skip_payload is defined
+
+        - name: End play for this host (backup already prepared; backup prep not needed)
+          ansible.builtin.meta: end_host
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.4-r9850.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Cleanup prepared marker after successful backup-bank write
+          ansible.builtin.raw: "rm -f '{{ fw_marker }}'"
+          changed_when: true
+          when:
+            - fw_marker is defined
+            - fw_marker | length > 0
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_outdoorbackup"
+              task_result: "backup_prepared"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Active (primary) firmux before: {{ bootbanks_active_firmux }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Success tag selection (ONLY CHANGE) --- (ONLY CHANGE) ---
+        - name: Build control queue payload for update-backup-prepared (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-backup-prepared (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-backup-prepared tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bank prepared for {{ fw_banner_repr }}.
+                Active unchanged; marker {{ fw_marker }}.
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state done (success) (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'done'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (update failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (update failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (update failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-outdoorbackup225.yml

@@ -0,0 +1,800 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.5-r9858.bin
+    firmware_sha256: "fd28e4ebef67f12a70152b9261bc7f1bdf8890bda9548d670dfc7bad98f98350"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read /var/run/bootbanks.json (active and backup firmux versions)
+          ansible.builtin.raw: "{{ pathprefix }} cat /var/run/bootbanks.json"
+          register: bootbanks_raw
+          changed_when: false
+
+        - name: Parse bootbanks.json
+          ansible.builtin.set_fact:
+            bootbanks_obj: "{{ bootbanks_raw.stdout | from_json }}"
+            bootbanks_active_firmux: "{{ (bootbanks_raw.stdout | from_json).active.firmux | default('unknown') }}"
+            bootbanks_backup_firmux: "{{ (bootbanks_raw.stdout | from_json).backup.firmux | default('unknown') }}"
+
+        - name: current versions (active primary and backup)
+          ansible.builtin.debug:
+            msg:
+              - "active (primary) firmux: {{ bootbanks_active_firmux }}"
+              - "backup firmux: {{ bootbanks_backup_firmux }}"
+
+        - name: Build control queue payload for skip journal (backup already prepared)
+          ansible.builtin.set_fact:
+            journal_skip_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bootbank already has {{ fw_banner_repr }} (backup.firmux={{ bootbanks_backup_firmux }}).
+                Skipping backup preparation (no flip, no reboot).
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+        - name: Bandaid | Pause before Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when: journal_skip_payload is defined
+
+        - name: Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_skip_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_skip_resp
+          changed_when: (rmq_journal_skip_resp.json is defined) and (rmq_journal_skip_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_skip_resp.status != 200) or
+            (rmq_journal_skip_resp.json is not defined) or
+            (not (rmq_journal_skip_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: journal_skip_payload is defined
+
+        - name: End play for this host (backup already prepared; backup prep not needed)
+          ansible.builtin.meta: end_host
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.5-r9858.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Cleanup prepared marker after successful backup-bank write
+          ansible.builtin.raw: "rm -f '{{ fw_marker }}'"
+          changed_when: true
+          when:
+            - fw_marker is defined
+            - fw_marker | length > 0
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_outdoorbackup"
+              task_result: "backup_prepared"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Active (primary) firmux before: {{ bootbanks_active_firmux }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Success tag selection (ONLY CHANGE) --- (ONLY CHANGE) ---
+        - name: Build control queue payload for update-backup-prepared (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-backup-prepared (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-backup-prepared tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bank prepared for {{ fw_banner_repr }}.
+                Active unchanged; marker {{ fw_marker }}.
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state done (success) (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'done'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (update failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (update failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (update failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-outdoorbackup226-scheduler.yml

@@ -0,0 +1,156 @@
+# update-outdoorbackup-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-outdoorbackup224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+      
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-outdoorbackup226.yml
+  vars:
+    is_run_by: "scheduler"
+
+- name: Phase 4 | Scheduled success only | Chain sot-updater-scheduler
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks:
+    - name: Scheduled success | Set action_next to sot-updater-scheduler
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success | Publish sot-updater-scheduler work message
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/update-outdoorbackup226.yml

@@ -0,0 +1,800 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.6-r9926.bin
+    firmware_sha256: "7a3cf094ec33e272468f6680409c9acb8eff662b1b1ef020c307852d1263221c"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read /var/run/bootbanks.json (active and backup firmux versions)
+          ansible.builtin.raw: "{{ pathprefix }} cat /var/run/bootbanks.json"
+          register: bootbanks_raw
+          changed_when: false
+
+        - name: Parse bootbanks.json
+          ansible.builtin.set_fact:
+            bootbanks_obj: "{{ bootbanks_raw.stdout | from_json }}"
+            bootbanks_active_firmux: "{{ (bootbanks_raw.stdout | from_json).active.firmux | default('unknown') }}"
+            bootbanks_backup_firmux: "{{ (bootbanks_raw.stdout | from_json).backup.firmux | default('unknown') }}"
+
+        - name: current versions (active primary and backup)
+          ansible.builtin.debug:
+            msg:
+              - "active (primary) firmux: {{ bootbanks_active_firmux }}"
+              - "backup firmux: {{ bootbanks_backup_firmux }}"
+
+        - name: Build control queue payload for skip journal (backup already prepared)
+          ansible.builtin.set_fact:
+            journal_skip_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bootbank already has {{ fw_banner_repr }} (backup.firmux={{ bootbanks_backup_firmux }}).
+                Skipping backup preparation (no flip, no reboot).
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+        - name: Bandaid | Pause before Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when: journal_skip_payload is defined
+
+        - name: Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_skip_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_skip_resp
+          changed_when: (rmq_journal_skip_resp.json is defined) and (rmq_journal_skip_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_skip_resp.status != 200) or
+            (rmq_journal_skip_resp.json is not defined) or
+            (not (rmq_journal_skip_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: journal_skip_payload is defined
+
+        - name: End play for this host (backup already prepared; backup prep not needed)
+          ansible.builtin.meta: end_host
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.6-r9926.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Cleanup prepared marker after successful backup-bank write
+          ansible.builtin.raw: "rm -f '{{ fw_marker }}'"
+          changed_when: true
+          when:
+            - fw_marker is defined
+            - fw_marker | length > 0
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_outdoorbackup"
+              task_result: "backup_prepared"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Active (primary) firmux before: {{ bootbanks_active_firmux }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Success tag selection (ONLY CHANGE) --- (ONLY CHANGE) ---
+        - name: Build control queue payload for update-backup-prepared (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-backup-prepared (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-backup-prepared tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bank prepared for {{ fw_banner_repr }}.
+                Active unchanged; marker {{ fw_marker }}.
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state done (success) (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'done'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (update failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (update failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (update failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-reboot-scheduler.yml

@@ -0,0 +1,97 @@
+# update-reboot-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-rebootin224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-rebootin224.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-reboot.yml

@@ -1,4 +1,4 @@
 ---
 # update-reboot.yml — thin wrapper, no var forwarding. 
 # Delegates entirely to the unified updater.
-- import_playbook: update-rebootin222.yml
+- import_playbook: update-rebootin224.yml

files/ansible-playbooks/update-reboot225-scheduler.yml

@@ -0,0 +1,100 @@
+# update-reboot-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-rebootin224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-rebootin225.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-reboot226-scheduler.yml

@@ -0,0 +1,100 @@
+# update-reboot-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-rebootin224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-rebootin226.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-rebootin223.yml

@@ -0,0 +1,809 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.3-r9800.bin
+    firmware_sha256: "5c06496a896831c5548988ac2575a0a84bb9054a769159dec354e1f996c252e4"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.3-r9800.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: "{{ au_delay_sec | default(300) }}"
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-upgrade payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterupgrade_payload={{ afterupgrade_payload | to_nice_json }}"
+              - "au_delay_sec={{ au_delay_sec }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost

files/ansible-playbooks/wifidebug-scheduler.yml

@@ -0,0 +1,244 @@
+# wifidebug-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run wifidebug17.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: wifidebug17.yml
+  vars:
+    is_run_by: "scheduler"
+
+
+- name: Phase 4 | NetBox wrapup and chaining for wifidebug
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: Wrapper compute wifidebug outcome
+      set_fact:
+        inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+        wifidebug_ok: "{{ wifidebug_success | default(false) | bool }}"
+        wifidebug_status: "{{ wifidebug_result_status | default('UNKNOWN') }}"
+      changed_when: false
+
+    - name: Wrapper pause before clear action_next_timestamp
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper clear action_next_timestamp
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before set action_state
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_state
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': (wifidebug_ok | ternary('done','failed')) } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before chaining to sot updater
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_next to sot-updater-scheduler
+      when: wifidebug_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper publish sot-updater-scheduler work message
+      when: wifidebug_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before final journal
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper final journal report
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'journal_add', 'task_result': (wifidebug_ok | ternary('wifidebug: successfully deployed','wifidebug: deployment failed')) ~ ' (' ~ wifidebug_status ~ ')' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/wifidebug17.yml

@@ -387,6 +387,12 @@
           set_fact:
             inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
 
+        - name: Expose outcome facts for scheduler wrapper
+          set_fact:
+            wifidebug_result_status: "{{ result_status | default('UNKNOWN') }}"
+            wifidebug_success: "{{ (result_status | default('')) in ['SUCCESS_DEPLOYED','SUCCESS_NO_CHANGE'] }}"
+          changed_when: false
+
         # (a) Set custom field wifidebug -> v15 on control queue
 
         - name: Build custom-field payload (wifidebug -> version)

files/files/connstats-crond-root

@@ -0,0 +1 @@
+*/10 * * * * /root/connstats.sh --debug --always-find-offset  >/dev/null 2>&1

files/files/connstats.sh

@@ -0,0 +1,678 @@
+#!/bin/sh
+# connstats.sh
+# Log-derived Wi-Fi connection counters per run (intended for cron every 10 minutes).
+# Writes one summary line per run to /root/connstats.log
+# Optional: --debug for chatty troubleshooting output in the same log.
+# Optional: --always-find-offset to calibrate syslog header time via a logger marker
+#           and (on fresh start / rotation / truncation) build a true last-10-min slice.
+
+SCRIPTVERSION=v1
+
+LOG_SRC="/var/log/messages"
+OUT_LOG="/root/connstats.log"
+STATE="/tmp/connstats.state"
+TMPDIR="/tmp"
+
+# How many tail lines to scan when we have no state (first run / after reboot / rotation)
+BOOTSTRAP_BACKFILL_LINES=2000
+
+# Used only with --always-find-offset during bootstrap-like runs
+BOOTSTRAP_TIME_WINDOW_SEC=600
+BOOTSTRAP_TIME_TAIL_LINES=10000
+
+DEBUG=0
+ALWAYS_FIND_OFFSET=0
+
+for arg in "$@"; do
+  case "$arg" in
+    --debug) DEBUG=1 ;;
+    --always-find-offset) ALWAYS_FIND_OFFSET=1 ;;
+    -h|--help)
+      echo "Usage: $0 [--debug] [--always-find-offset]"
+      exit 0
+      ;;
+    *)
+      echo "Unknown arg: $arg" >&2
+      echo "Usage: $0 [--debug] [--always-find-offset]" >&2
+      exit 2
+      ;;
+  esac
+done
+
+umask 077
+
+ts_iso() {
+  # ISO-ish timestamp; works on BusyBox and GNU date
+  date "+%Y-%m-%dT%H:%M:%S%z"
+}
+
+log() {
+  # Always append to OUT_LOG
+  echo "[$(ts_iso)] $*" >> "$OUT_LOG"
+}
+
+# Generate a short random token (8 chars) for marker grep
+rand_token() {
+  if [ -r /dev/urandom ]; then
+    tr -dc 'a-z0-9' < /dev/urandom 2>/dev/null | head -c 8
+  else
+    echo "$(date -u +%s 2>/dev/null)$$" | tr -dc 'a-z0-9' | tail -c 8
+  fi
+}
+
+# Parse syslog header timestamp (Mon DD HH:MM:SS) into epoch seconds (UTC) using awk mktime()
+# Expects typical line like: "<13> Jan 31 12:31:00 root[-] [notice]: message"
+syslog_header_to_epoch_utc() {
+  echo "$1" | TZ=UTC awk '
+    function mon2num(m) {
+      if (m=="Jan") return 1
+      if (m=="Feb") return 2
+      if (m=="Mar") return 3
+      if (m=="Apr") return 4
+      if (m=="May") return 5
+      if (m=="Jun") return 6
+      if (m=="Jul") return 7
+      if (m=="Aug") return 8
+      if (m=="Sep") return 9
+      if (m=="Oct") return 10
+      if (m=="Nov") return 11
+      if (m=="Dec") return 12
+      return 0
+    }
+    {
+      mon=$2; day=$3; tod=$4
+      year=strftime("%Y")
+      m=mon2num(mon)
+      if (m==0) { print ""; exit 1 }
+      split(tod, t, ":")
+      if (length(t) != 3) { print ""; exit 1 }
+      hh=t[1]+0; mm=t[2]+0; ss=t[3]+0
+      d=day+0
+      print mktime(sprintf("%d %02d %02d %02d %02d %02d", year, m, d, hh, mm, ss))
+    }
+  '
+}
+
+# Compute offset_sec = payload_epoch_utc - syslog_header_epoch_utc
+# Returns offset seconds on stdout (blank on failure)
+find_time_offset_sec() {
+  token="$(rand_token)"
+  now_epoch="$(date -u +%s 2>/dev/null | tr -d ' ')"
+  [ -z "$now_epoch" ] && now_epoch=0
+
+  logger "connstat calculation started. issueing time marker $token epoch=$now_epoch" 2>/dev/null
+
+  found_line=""
+  i=0
+  while [ $i -lt 3 ]; do
+    found_line="$(tail -n 300 "$LOG_SRC" 2>/dev/null | grep "$token" | tail -n 1)"
+    [ -n "$found_line" ] && break
+    sleep 1
+    i=$((i + 1))
+  done
+
+  [ -z "$found_line" ] && { echo ""; return; }
+
+  hdr_epoch="$(syslog_header_to_epoch_utc "$found_line" | tr -d ' ')"
+  [ -z "$hdr_epoch" ] && { echo ""; return; }
+
+  payload_epoch="$(echo "$found_line" | awk '{
+    for (i=1; i<=NF; i++) {
+      if ($i ~ /^epoch=[0-9]+$/) { sub(/^epoch=/,"",$i); print $i; exit }
+    }
+  }' | tr -d ' ')"
+  [ -z "$payload_epoch" ] && payload_epoch="$now_epoch"
+
+  echo $((payload_epoch - hdr_epoch))
+}
+
+# Basic sanity checks
+if [ ! -r "$LOG_SRC" ]; then
+  log "ERROR log_src_unreadable path=$LOG_SRC"
+  exit 1
+fi
+
+# Get current inode + line count (line-based incremental read is most portable)
+cur_inode="$(ls -i "$LOG_SRC" 2>/dev/null | awk '{print $1}')"
+cur_lines="$(wc -l < "$LOG_SRC" 2>/dev/null | tr -d ' ')"
+
+if [ -z "$cur_inode" ] || [ -z "$cur_lines" ]; then
+  log "ERROR cannot_stat_log inode='$cur_inode' lines='$cur_lines'"
+  exit 1
+fi
+
+last_inode=""
+last_line=""
+
+if [ -f "$STATE" ]; then
+  # STATE format:
+  # inode=<num>
+  # line=<num>
+  last_inode="$(grep '^inode=' "$STATE" 2>/dev/null | head -n1 | cut -d= -f2)"
+  last_line="$(grep '^line=' "$STATE" 2>/dev/null | head -n1 | cut -d= -f2)"
+fi
+
+# Decide where to start reading
+start_line=""
+
+backfill_start() {
+  start_line=$((cur_lines - BOOTSTRAP_BACKFILL_LINES + 1))
+  [ "$start_line" -lt 1 ] && start_line=1
+}
+
+# Extract the slice
+tmp_slice="$TMPDIR/connstats.slice.$$"
+
+# Grep patterns (hostapd transition events)
+re_auth='hostapd.*IEEE 802\.11: authenticated'
+re_assoc='hostapd.*IEEE 802\.11: associated'
+re_disassoc='hostapd.*IEEE 802\.11: disassociated'
+
+# Kernel extras for "seen_any" breadth (optional but useful)
+re_kernel_seen='kernel.*(station kicked out|station timed out)'
+
+# Are we in a bootstrap-like condition (no state OR rotation/truncation)?
+bootstrap_like=0
+if [ -z "$last_inode" ] || [ -z "$last_line" ]; then
+  bootstrap_like=1
+else
+  if [ "$cur_inode" != "$last_inode" ] || [ "$cur_lines" -lt "$last_line" ]; then
+    bootstrap_like=1
+  fi
+fi
+
+# Bootstrap handling
+if [ "$ALWAYS_FIND_OFFSET" -eq 1 ] && [ "$bootstrap_like" -eq 1 ]; then
+  offset_sec="$(find_time_offset_sec)"
+
+  if [ -n "$offset_sec" ]; then
+    now_epoch="$(date -u +%s 2>/dev/null | tr -d ' ')"
+    [ -z "$now_epoch" ] && now_epoch=0
+    cutoff_epoch=$((now_epoch - BOOTSTRAP_TIME_WINDOW_SEC))
+
+    if [ $DEBUG -eq 1 ]; then
+      log "DEBUG bootstrap_time_window_last_10m offset_sec=$offset_sec cutoff_epoch=$cutoff_epoch tail_lines=$BOOTSTRAP_TIME_TAIL_LINES"
+    fi
+
+    # Build slice = last-10-min relevant lines (hostapd transitions + kernel station kicked/timed out)
+    # using (syslog_header_epoch + offset_sec) >= cutoff_epoch
+    tail -n "$BOOTSTRAP_TIME_TAIL_LINES" "$LOG_SRC" 2>/dev/null \
+      | TZ=UTC awk -v cutoff="$cutoff_epoch" -v off="$offset_sec" '
+        function mon2num(m) {
+          if (m=="Jan") return 1
+          if (m=="Feb") return 2
+          if (m=="Mar") return 3
+          if (m=="Apr") return 4
+          if (m=="May") return 5
+          if (m=="Jun") return 6
+          if (m=="Jul") return 7
+          if (m=="Aug") return 8
+          if (m=="Sep") return 9
+          if (m=="Oct") return 10
+          if (m=="Nov") return 11
+          if (m=="Dec") return 12
+          return 0
+        }
+        function header_epoch(   mon,day,tod,year,m,hh,mm,ss,t) {
+          mon=$2; day=$3; tod=$4
+          year=strftime("%Y")
+          m=mon2num(mon)
+          if (m==0) return -1
+          split(tod, t, ":")
+          if (length(t) != 3) return -1
+          hh=t[1]+0; mm=t[2]+0; ss=t[3]+0
+          return mktime(sprintf("%d %02d %02d %02d %02d %02d", year, m, (day+0), hh, mm, ss))
+        }
+        {
+          line=$0
+          if (line ~ /hostapd.*IEEE 802\.11: (authenticated|associated|disassociated)/ ||
+              line ~ /kernel.*(station kicked out|station timed out)/) {
+            he=header_epoch()
+            if (he < 0) next
+            corr=he + off
+            if (corr >= cutoff) print line
+          }
+        }
+      ' > "$tmp_slice" 2>/dev/null
+
+    start_line="(time_window_last_10m)"
+  else
+    # Offset calc failed -> fall back to old bootstrap behavior
+    backfill_start
+    if [ $DEBUG -eq 1 ]; then
+      log "DEBUG time_offset_failed_fallback_to_backfill cur_inode=$cur_inode cur_lines=$cur_lines start_line=$start_line backfill_lines=$BOOTSTRAP_BACKFILL_LINES"
+    fi
+    sed -n "${start_line},\$p" "$LOG_SRC" > "$tmp_slice" 2>/dev/null
+  fi
+
+else
+  # Original behavior (unchanged)
+  if [ -z "$last_inode" ] || [ -z "$last_line" ]; then
+    backfill_start
+    if [ $DEBUG -eq 1 ]; then
+      log "DEBUG bootstrap_no_state_backfill cur_inode=$cur_inode cur_lines=$cur_lines start_line=$start_line backfill_lines=$BOOTSTRAP_BACKFILL_LINES"
+    fi
+  else
+    # Rotation / truncation detection
+    if [ "$cur_inode" != "$last_inode" ] || [ "$cur_lines" -lt "$last_line" ]; then
+      backfill_start
+      if [ $DEBUG -eq 1 ]; then
+        log "DEBUG bootstrap_rotation_or_truncate_backfill last_inode=$last_inode cur_inode=$cur_inode last_line=$last_line cur_lines=$cur_lines start_line=$start_line backfill_lines=$BOOTSTRAP_BACKFILL_LINES"
+      fi
+    else
+      start_line=$((last_line + 1))
+      if [ $DEBUG -eq 1 ]; then
+        log "DEBUG incremental last_line=$last_line cur_lines=$cur_lines start_line=$start_line"
+      fi
+    fi
+  fi
+
+  sed -n "${start_line},\$p" "$LOG_SRC" > "$tmp_slice" 2>/dev/null
+fi
+
+slice_lines="$(wc -l < "$tmp_slice" 2>/dev/null | tr -d ' ')"
+[ -z "$slice_lines" ] && slice_lines=0
+
+if [ $DEBUG -eq 1 ]; then
+  log "DEBUG slice_path=$tmp_slice slice_lines=$slice_lines"
+fi
+
+# Totals (MAC repeats allowed)
+auth_events_total="$(grep -E "$re_auth" "$tmp_slice" | wc -l | tr -d ' ')"
+assoc_events_total="$(grep -E "$re_assoc" "$tmp_slice" | wc -l | tr -d ' ')"
+disassoc_events_total="$(grep -E "$re_disassoc" "$tmp_slice" | wc -l | tr -d ' ')"
+
+# Extract MAC right after token "STA" (from hostapd lines)
+extract_sta_macs() {
+  awk '
+    {
+      for (i=1; i<=NF; i++) {
+        if ($i=="STA") {
+          print $(i+1);
+          break;
+        }
+      }
+    }
+  '
+}
+
+# Extract MACs in brackets: [aa:bb:cc:dd:ee:ff]
+extract_bracket_macs() {
+  grep -oE '\[[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}\]' | tr -d '[]'
+}
+
+# Count unique valid MACs from stdin
+uniq_count() {
+  grep -E '^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$' \
+  | sort -u \
+  | wc -l | tr -d ' '
+}
+
+# Unique MACs per event type (hostapd)
+unique_macs_authenticated="$(
+  grep -E "$re_auth" "$tmp_slice" \
+  | extract_sta_macs \
+  | uniq_count
+)"
+
+unique_macs_associated="$(
+  grep -E "$re_assoc" "$tmp_slice" \
+  | extract_sta_macs \
+  | uniq_count
+)"
+
+unique_macs_disassociated="$(
+  grep -E "$re_disassoc" "$tmp_slice" \
+  | extract_sta_macs \
+  | uniq_count
+)"
+
+# Unique MACs seen in "any relevant activity" (union: auth/assoc/disassoc + selected kernel)
+unique_macs_seen_any="$(
+  (
+    grep -E 'hostapd.*IEEE 802\.11: (authenticated|associated|disassociated)' "$tmp_slice" \
+      | extract_sta_macs
+
+    grep -E "$re_kernel_seen" "$tmp_slice" \
+      | extract_bracket_macs
+  ) | uniq_count
+)"
+
+########################################################################
+# B-class metrics (ephemeral state in /tmp; reboot loss accepted)
+########################################################################
+
+SESSION_STATE="/tmp/connstats.sessions.state"
+SESSION_STATE_TMP="$TMPDIR/connstats.sessions.state.$$"
+SESSION_STATE_TTL_SEC=21600
+AUTH_ASSOC_MAX_SAMPLES=50
+
+now_epoch_b="$(date -u +%s 2>/dev/null | tr -d ' ')"
+[ -z "$now_epoch_b" ] && now_epoch_b=0
+
+# Output file (temporary, removed each run)
+B_OUT_TMP="$TMPDIR/connstats.b.out.$$"
+
+TZ=UTC awk -v state_in="$SESSION_STATE" \
+          -v state_out="$SESSION_STATE_TMP" \
+          -v now="$now_epoch_b" \
+          -v ttl="$SESSION_STATE_TTL_SEC" \
+          -v maxs="$AUTH_ASSOC_MAX_SAMPLES" '
+  function mon2num(m) {
+    if (m=="Jan") return 1
+    if (m=="Feb") return 2
+    if (m=="Mar") return 3
+    if (m=="Apr") return 4
+    if (m=="May") return 5
+    if (m=="Jun") return 6
+    if (m=="Jul") return 7
+    if (m=="Aug") return 8
+    if (m=="Sep") return 9
+    if (m=="Oct") return 10
+    if (m=="Nov") return 11
+    if (m=="Dec") return 12
+    return 0
+  }
+  function header_epoch(   mon,day,tod,year,m,hh,mm,ss,t) {
+    mon=$2; day=$3; tod=$4
+    year=strftime("%Y")
+    m=mon2num(mon)
+    if (m==0) return -1
+    split(tod, t, ":")
+    if (length(t) != 3) return -1
+    hh=t[1]+0; mm=t[2]+0; ss=t[3]+0
+    return mktime(sprintf("%d %02d %02d %02d %02d %02d", year, m, (day+0), hh, mm, ss))
+  }
+  function extract_sta_mac(   i) {
+    for (i=1; i<=NF; i++) {
+      if ($i=="STA") return $(i+1)
+    }
+    return ""
+  }
+  function mac_valid(m) {
+    return (m ~ /^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$/)
+  }
+
+  BEGIN {
+    if (state_in != "") {
+      while ((getline line < state_in) > 0) {
+        n = split(line, a, " ")
+        if (n >= 2) {
+          m=a[1]; e=a[2]+0
+          if (mac_valid(m) && e > 0) last_auth[m]=e
+        }
+      }
+      close(state_in)
+    }
+  }
+
+  {
+    line=$0
+    if (line !~ /hostapd.*IEEE 802\.11: (authenticated|associated|disassociated)/) next
+
+    e = header_epoch()
+    if (e < 0) next
+
+    mac = extract_sta_mac()
+    if (!mac_valid(mac)) next
+
+    if (line ~ /IEEE 802\.11: authenticated/) {
+      last_auth[mac]=e
+      next
+    }
+
+    if (line ~ /IEEE 802\.11: associated/) {
+      if (mac in last_auth) {
+        d = e - last_auth[mac]
+        if (d >= 0) {
+          ms = d * 1000
+          if (sample_count < maxs) {
+            if (samples == "") samples = ms
+            else samples = samples "," ms
+            sample_count++
+          }
+        }
+      }
+      next
+    }
+
+    if (line ~ /IEEE 802\.11: disassociated/) {
+      if (mac in last_auth) {
+        d = e - last_auth[mac]
+        if (d >= 0) {
+          full_sessions_total++
+          sum_len += d
+          if (d < 60) early_drop_sessions_total++
+          delete last_auth[mac]
+        }
+      }
+      next
+    }
+  }
+
+  END {
+    if (full_sessions_total > 0) avg_session_length = int((sum_len / full_sessions_total) + 0.5)
+    else avg_session_length = 0
+
+    if (state_out != "") {
+      for (m in last_auth) {
+        e = last_auth[m] + 0
+        if (e > 0 && (now <= 0 || (e >= (now - ttl)))) {
+          print m " " e > state_out
+        }
+      }
+      close(state_out)
+    }
+
+    print full_sessions_total "\t" avg_session_length "\t" early_drop_sessions_total "\t" samples
+  }
+' "$tmp_slice" > "$B_OUT_TMP" 2>/dev/null
+
+# Default B fields
+full_sessions_total=0
+avg_session_length=0
+early_drop_sessions_total=0
+auth_to_assoc_time_ms_samples=""
+
+# Parse awk output
+if [ -s "$B_OUT_TMP" ]; then
+  b_out_line="$(head -n 1 "$B_OUT_TMP" 2>/dev/null)"
+  full_sessions_total="$(echo "$b_out_line" | awk -F'\t' '{print $1}' | tr -d ' ')"
+  avg_session_length="$(echo "$b_out_line" | awk -F'\t' '{print $2}' | tr -d ' ')"
+  early_drop_sessions_total="$(echo "$b_out_line" | awk -F'\t' '{print $3}' | tr -d ' ')"
+  auth_to_assoc_time_ms_samples="$(echo "$b_out_line" | awk -F'\t' '{print $4}')"
+fi
+
+rm -f "$B_OUT_TMP" 2>/dev/null
+
+[ -z "$full_sessions_total" ] && full_sessions_total=0
+[ -z "$avg_session_length" ] && avg_session_length=0
+[ -z "$early_drop_sessions_total" ] && early_drop_sessions_total=0
+
+# Move state into place (ephemeral but persistent across runs until reboot)
+if [ -f "$SESSION_STATE_TMP" ]; then
+  mv -f "$SESSION_STATE_TMP" "$SESSION_STATE" 2>/dev/null
+else
+  rm -f "$SESSION_STATE_TMP" 2>/dev/null
+fi
+
+########################################################################
+# C-class metrics (kernel kick reason codes 1..8)
+########################################################################
+
+# Only count kernel "station kicked out ..." (avoids PRS DRIVER_LOG "reason" lines)
+re_kick_reason_base='kernel.*station kicked out.*reason[[:space:]]+'
+
+reason_code_events_1="$(grep -E "${re_kick_reason_base}1([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_2="$(grep -E "${re_kick_reason_base}2([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_3="$(grep -E "${re_kick_reason_base}3([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_4="$(grep -E "${re_kick_reason_base}4([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_5="$(grep -E "${re_kick_reason_base}5([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_6="$(grep -E "${re_kick_reason_base}6([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_7="$(grep -E "${re_kick_reason_base}7([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_8="$(grep -E "${re_kick_reason_base}8([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+
+[ -z "$reason_code_events_1" ] && reason_code_events_1=0
+[ -z "$reason_code_events_2" ] && reason_code_events_2=0
+[ -z "$reason_code_events_3" ] && reason_code_events_3=0
+[ -z "$reason_code_events_4" ] && reason_code_events_4=0
+[ -z "$reason_code_events_5" ] && reason_code_events_5=0
+[ -z "$reason_code_events_6" ] && reason_code_events_6=0
+[ -z "$reason_code_events_7" ] && reason_code_events_7=0
+[ -z "$reason_code_events_8" ] && reason_code_events_8=0
+
+########################################################################
+# D-class metrics (flaps between ath0 and ath1 within the bucket)
+########################################################################
+
+D_OUT_TMP="$TMPDIR/connstats.d.out.$$"
+
+awk '
+  function extract_sta_mac(   i) {
+    for (i=1; i<=NF; i++) if ($i=="STA") return $(i+1)
+    return ""
+  }
+  function mac_valid(m) {
+    return (m ~ /^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$/)
+  }
+  function extract_iface(   i, v) {
+    for (i=1; i<=NF; i++) {
+      v = $i
+      if (v ~ /^ath[0-9]+:$/) { sub(/:$/,"",v); return v }
+    }
+    return ""
+  }
+
+  {
+    line=$0
+    if (line !~ /hostapd.*IEEE 802\.11: associated/) next
+
+    iface = extract_iface()
+    if (iface != "ath0" && iface != "ath1") next
+
+    mac = extract_sta_mac()
+    if (!mac_valid(mac)) next
+
+    if (mac in last_iface) {
+      prev = last_iface[mac]
+      if (prev != iface) {
+        if (prev=="ath0" && iface=="ath1") {
+          flap_events_0_to_1++
+          uniq_0_to_1[mac]=1
+        } else if (prev=="ath1" && iface=="ath0") {
+          flap_events_1_to_0++
+          uniq_1_to_0[mac]=1
+        }
+      }
+    }
+
+    last_iface[mac]=iface
+  }
+
+  END {
+    for (m in uniq_0_to_1) unique_0_to_1++
+    for (m in uniq_1_to_0) unique_1_to_0++
+
+    if (unique_0_to_1+0 < 0) unique_0_to_1=0
+    if (flap_events_0_to_1+0 < 0) flap_events_0_to_1=0
+    if (unique_1_to_0+0 < 0) unique_1_to_0=0
+    if (flap_events_1_to_0+0 < 0) flap_events_1_to_0=0
+
+    print unique_0_to_1 "\t" flap_events_0_to_1 "\t" unique_1_to_0 "\t" flap_events_1_to_0
+  }
+' "$tmp_slice" > "$D_OUT_TMP" 2>/dev/null
+
+unique_macs_flap_ath0_to_ath1=0
+flap_events_total_ath0_to_ath1=0
+unique_macs_flap_ath1_to_ath0=0
+flap_events_total_ath1_to_ath0=0
+
+if [ -s "$D_OUT_TMP" ]; then
+  d_out_line="$(head -n 1 "$D_OUT_TMP" 2>/dev/null)"
+  unique_macs_flap_ath0_to_ath1="$(echo "$d_out_line" | awk -F'\t' '{print $1}' | tr -d ' ')"
+  flap_events_total_ath0_to_ath1="$(echo "$d_out_line" | awk -F'\t' '{print $2}' | tr -d ' ')"
+  unique_macs_flap_ath1_to_ath0="$(echo "$d_out_line" | awk -F'\t' '{print $3}' | tr -d ' ')"
+  flap_events_total_ath1_to_ath0="$(echo "$d_out_line" | awk -F'\t' '{print $4}' | tr -d ' ')"
+fi
+
+rm -f "$D_OUT_TMP" 2>/dev/null
+
+[ -z "$unique_macs_flap_ath0_to_ath1" ] && unique_macs_flap_ath0_to_ath1=0
+[ -z "$flap_events_total_ath0_to_ath1" ] && flap_events_total_ath0_to_ath1=0
+[ -z "$unique_macs_flap_ath1_to_ath0" ] && unique_macs_flap_ath1_to_ath0=0
+[ -z "$flap_events_total_ath1_to_ath0" ] && flap_events_total_ath1_to_ath0=0
+
+########################################################################
+# Emit one summary line
+########################################################################
+log "connstats bucket_run slice_lines=$slice_lines unique_macs_seen_any=$unique_macs_seen_any unique_macs_authenticated=$unique_macs_authenticated auth_events_total=$auth_events_total unique_macs_associated=$unique_macs_associated assoc_events_total=$assoc_events_total unique_macs_disassociated=$unique_macs_disassociated disassoc_events_total=$disassoc_events_total full_sessions_total=$full_sessions_total avg_session_length=$avg_session_length early_drop_sessions_total=$early_drop_sessions_total auth_to_assoc_time_ms_samples=$auth_to_assoc_time_ms_samples reason_code_events_1=$reason_code_events_1 reason_code_events_2=$reason_code_events_2 reason_code_events_3=$reason_code_events_3 reason_code_events_4=$reason_code_events_4 reason_code_events_5=$reason_code_events_5 reason_code_events_6=$reason_code_events_6 reason_code_events_7=$reason_code_events_7 reason_code_events_8=$reason_code_events_8 unique_macs_flap_ath0_to_ath1=$unique_macs_flap_ath0_to_ath1 flap_events_total_ath0_to_ath1=$flap_events_total_ath0_to_ath1 unique_macs_flap_ath1_to_ath0=$unique_macs_flap_ath1_to_ath0 flap_events_total_ath1_to_ath0=$flap_events_total_ath1_to_ath0"
+
+if [ $DEBUG -eq 1 ]; then
+  log "DEBUG dump_auth_lines_begin"
+  grep -E "$re_auth" "$tmp_slice" | sed 's/^/DEBUG AUTH: /' >> "$OUT_LOG"
+  log "DEBUG dump_assoc_lines_begin"
+  grep -E "$re_assoc" "$tmp_slice" | sed 's/^/DEBUG ASSOC: /' >> "$OUT_LOG"
+  log "DEBUG dump_disassoc_lines_begin"
+  grep -E "$re_disassoc" "$tmp_slice" | sed 's/^/DEBUG DISASSOC: /' >> "$OUT_LOG"
+
+  log "DEBUG dump_kick_reason_lines_begin"
+  grep -E "${re_kick_reason_base}[1-8]([^0-9]|$)" "$tmp_slice" | sed 's/^/DEBUG KICK: /' >> "$OUT_LOG"
+
+  log "DEBUG dump_flap_assoc_lines_begin"
+  grep -E 'hostapd.*IEEE 802\.11: associated' "$tmp_slice" | sed 's/^/DEBUG FLAP_ASSOC: /' >> "$OUT_LOG"
+
+  log "DEBUG dump_flap_events_begin"
+  awk '
+    function extract_sta_mac(   i) { for (i=1; i<=NF; i++) if ($i=="STA") return $(i+1); return "" }
+    function mac_valid(m) { return (m ~ /^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$/) }
+    function extract_iface(   i, v) {
+      for (i=1; i<=NF; i++) { v=$i; if (v ~ /^ath[0-9]+:$/) { sub(/:$/,"",v); return v } }
+      return ""
+    }
+    {
+      if ($0 !~ /hostapd.*IEEE 802\.11: associated/) next
+      iface=extract_iface()
+      if (iface!="ath0" && iface!="ath1") next
+      mac=extract_sta_mac()
+      if (!mac_valid(mac)) next
+      if (mac in last_iface) {
+        prev=last_iface[mac]
+        if (prev!=iface) {
+          if ((prev=="ath0" && iface=="ath1") || (prev=="ath1" && iface=="ath0")) {
+            print "mac=" mac " from=" prev " to=" iface
+          }
+        }
+      }
+      last_iface[mac]=iface
+    }
+  ' "$tmp_slice" | sed 's/^/DEBUG FLAP: /' >> "$OUT_LOG"
+fi
+
+# ---------------------------------------------------------------------
+# Send one syslog message (wifidebug-style tag), after random delay
+# ---------------------------------------------------------------------
+
+# Hostname (as in wifidebug)
+HOSTNAME="$(cat /proc/sys/kernel/hostname 2>/dev/null)"
+[ -z "$HOSTNAME" ] && HOSTNAME="$(hostname 2>/dev/null)"
+[ -z "$HOSTNAME" ] && HOSTNAME="unknown"
+
+# Random delay 3..20 seconds (right before sending)
+delay="$(hexdump -n2 -e '/2 "%u"' /dev/urandom 2>/dev/null)"
+case "$delay" in (''|*[!0-9]*) delay=0;; esac
+delay=$(( (delay % 18) + 3 ))
+sleep "$delay"
+
+# Exact stats payload (same fields as file line, no extra prefixes)
+MSG="slice_lines=$slice_lines unique_macs_seen_any=$unique_macs_seen_any unique_macs_authenticated=$unique_macs_authenticated auth_events_total=$auth_events_total unique_macs_associated=$unique_macs_associated assoc_events_total=$assoc_events_total unique_macs_disassociated=$unique_macs_disassociated disassoc_events_total=$disassoc_events_total full_sessions_total=$full_sessions_total avg_session_length=$avg_session_length early_drop_sessions_total=$early_drop_sessions_total auth_to_assoc_time_ms_samples=$auth_to_assoc_time_ms_samples reason_code_events_1=$reason_code_events_1 reason_code_events_2=$reason_code_events_2 reason_code_events_3=$reason_code_events_3 reason_code_events_4=$reason_code_events_4 reason_code_events_5=$reason_code_events_5 reason_code_events_6=$reason_code_events_6 reason_code_events_7=$reason_code_events_7 reason_code_events_8=$reason_code_events_8 unique_macs_flap_ath0_to_ath1=$unique_macs_flap_ath0_to_ath1 flap_events_total_ath0_to_ath1=$flap_events_total_ath0_to_ath1 unique_macs_flap_ath1_to_ath0=$unique_macs_flap_ath1_to_ath0 flap_events_total_ath1_to_ath0=$flap_events_total_ath1_to_ath0"
+
+logger -t "connstats|${HOSTNAME}" "$MSG" 2>/dev/null
+
+# Update state to current end-of-file line count and inode
+{
+  echo "inode=$cur_inode"
+  echo "line=$cur_lines"
+} > "$STATE"
+
+rm -f "$tmp_slice" 2>/dev/null
+exit 0

files/nb_onedevice_update-old.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""
+nb_sync_one_device.py
+
+Update a single NetBox device from Cloud by hostname — ONLY if device is online.
+
+Behavior
+- Liveness gate via /v1/devices/{cloud_id}/liveness (no changes if offline).
+- Updates NetBox fields from Cloud:
+  custom_fields.fw_version ← firmwareVersion
+  custom_fields.nodeName   ← nodeName
+  custom_fields.sectorName ← sectorName
+  custom_fields.smallCellName ← smallCellName
+  serial ← serialNumber
+- IP handling:
+  If Cloud ipAddress is valid (not None/""/"0.0.0.0"):
+    ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
+    then PRUNE all other IPs on this device (default).
+  If Cloud ipAddress is placeholder/invalid:
+    skip IP changes and do not prune.
+- --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
+- NEW: Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+
+Exit codes:
+  0 = success
+  1 = not found / missing data / offline
+  3 = network/HTTP error
+  4 = NetBox update error
+"""
+
+import sys
+import json
+import argparse
+from typing import Optional, Union, List, Dict
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# ------------ HARD-CODED CONFIG (per Pavel) ------------
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v1/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
+
+REQ_TIMEOUT = 30
+CHATTY = False
+
+# ------------ HTTP utilities ------------
+def _new_session() -> requests.Session:
+    s = requests.Session()
+    retries = Retry(
+        total=3, connect=3, read=3, status=3,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+        respect_retry_after_header=True,
+    )
+    adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
+    s.mount('http://', adapter)
+    s.mount('https://', adapter)
+    s.headers['Accept'] = 'application/json'
+    return s
+
+S_NB = _new_session()
+S_CL = _new_session()
+
+# ------------ Logging / status helpers ------------
+def log(msg: str):
+    if CHATTY:
+        print(msg, file=sys.stderr)
+
+def die(code: int, msg: str):
+    # Single-line result: stdout on success, stderr on failure
+    print(msg, file=sys.stdout if code == 0 else sys.stderr)
+    raise SystemExit(code)
+
+# ------------ NetBox API ------------
+class NetBox:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip('/')
+        self.token = token
+
+    def _h(self):
+        return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def get_device_by_name(self, name: str) -> Optional[dict]:
+        log(f"NB: lookup device by name {name}")
+        r = S_NB.get(self._url("/api/dcim/devices/"), headers=self._h(),
+                     params={"name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET devices name={name} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def get_device(self, dev_id: int) -> dict:
+        log(f"NB: fetch device id={dev_id}")
+        r = S_NB.get(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET device id={dev_id} HTTP={r.status_code}")
+        return r.json()
+
+    def patch_device(self, dev_id: int, patch: dict) -> None:
+        if not patch:
+            log("NB: no device patch needed")
+            return
+        log(f"NB: patch device id={dev_id} keys={list(patch.keys())}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps(patch), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL patch device dev={dev_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                     params={"device_id": dev_id, "name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET interfaces device_id={dev_id} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0]["id"] if res else None
+
+    def ensure_eth0(self, dev_id: int) -> int:
+        ifid = self.get_iface_id(dev_id, "eth0")
+        log(f"NB: ensure eth0 (current id={ifid})")
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = S_NB.post(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        if r.status_code == 400:
+            # race: read again
+            ifid = self.get_iface_id(dev_id, "eth0")
+            if ifid:
+                return ifid
+        die(4, f"FAIL create eth0 HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_ip_by_addr(self, addr: str) -> Optional[dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"address": f"{addr}/32"}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET ip-addresses addr={addr} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def create_ip_for_iface(self, addr: str, iface_id: int) -> int:
+        payload = {
+            "address": f"{addr}/32",
+            "status": "active",
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        }
+        r = S_NB.post(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        die(4, f"FAIL create IP {addr} HTTP={r.status_code} body={r.text[:200]}")
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
+        r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
+                       data=json.dumps({
+                           "assigned_object_type": "dcim.interface",
+                           "assigned_object_id": iface_id
+                       }),
+                       timeout=REQ_TIMEOUT)
+        return 200 <= r.status_code < 300
+
+    def device_set_primary_ip4(self, dev_id: int, ip_id: int) -> None:
+        log(f"NB: set primary_ip4 dev={dev_id} -> ip_id={ip_id}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps({"primary_ip4": ip_id}), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL set primary_ip4 dev={dev_id} ip_id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = S_NB.get(self._url(f"/api/dcim/interfaces/{iface_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return (r.json().get("device") or {}).get("id")
+        return None
+
+    # --- Helpers for pruning ---
+    def list_device_ips(self, dev_id: int) -> List[Dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return r.json().get("results") or []
+        ips: List[Dict] = []
+        r2 = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r2.status_code == 200:
+            for iface in (r2.json().get("results") or []):
+                ifid = iface.get("id")
+                r3 = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                              params={"assigned_object_type": "dcim.interface",
+                                      "assigned_object_id": ifid, "limit": 1000}, timeout=REQ_TIMEOUT)
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get("results") or [])
+        return ips
+
+    def delete_ip(self, ip_id: int) -> None:
+        log(f"NB: delete IP id={ip_id}")
+        r = S_NB.delete(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300 or r.status_code == 204):
+            die(4, f"FAIL delete IP id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+# ------------ Cloud API ------------
+class Cloud:
+    def __init__(self, base: str, bearer: str):
+        self.base = base.rstrip("/")
+        self.bearer = bearer
+
+    def _h(self):
+        return {"Authorization": f"Bearer {self.bearer}", "Accept": "application/json"}
+
+    def device_detail(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}"
+        log(f"CL: fetch detail cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id} HTTP={r.status_code}")
+        return r.json()
+
+    def device_liveness(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}/liveness"
+        log(f"CL: fetch liveness cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
+        return r.json()
+
+# ------------ Core ------------
+def _primary_ip4_text(dev_json: dict) -> Optional[str]:
+    p = dev_json.get("primary_ip4") or {}
+    addr = p.get("address")
+    if isinstance(addr, str) and addr.endswith("/32"):
+        return addr[:-3]
+    return addr
+
+def run(hostname: str) -> None:
+    nb = NetBox(NB_URL, NB_TOKEN)
+    cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+
+    dev = nb.get_device_by_name(hostname)
+    if not dev:
+        die(1, f"FAIL {hostname} not found in NetBox")
+
+    dev_id = dev["id"]
+    dev_full = nb.get_device(dev_id)
+    cf = dev_full.get("custom_fields") or {}
+
+    # NEW: log upgrade command if present
+    upgrade_cmd = cf.get("upgrade_cmd")
+    log(f"NB: upgrade_cmd={upgrade_cmd}")
+
+    cloud_id = cf.get("cloud_id")
+    if cloud_id in (None, "", "null"):
+        die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
+
+    # 1) Liveness gate
+    live = cl.device_liveness(cloud_id)
+    if not bool(live.get("isConnected")):
+        die(1, f"FAIL {hostname} cloud_id={cloud_id} device is not online")
+
+    current_nb_ip = _primary_ip4_text(dev_full)
+
+    # 2) Cloud detail
+    d = cl.device_detail(cloud_id)
+    fw = d.get("firmwareVersion") or d.get("version")
+    ip_from_cloud = (d.get("ipAddress") or "").strip() if isinstance(d.get("ipAddress"), str) else d.get("ipAddress")
+    node = d.get("nodeName")
+    sector = d.get("sectorName")
+    small = d.get("smallCellName")
+    serial = d.get("serialNumber")
+
+    if not fw:
+        die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
+
+    # 3) NetBox patch (idempotent)
+    cf_patch = {}
+    if cf.get("fw_version") != fw:
+        cf_patch["fw_version"] = fw
+        log(f"CF: fw_version -> {fw}")
+    if node and cf.get("nodeName") != node:
+        cf_patch["nodeName"] = node
+        log(f"CF: nodeName -> {node}")
+    if sector and cf.get("sectorName") != sector:
+        cf_patch["sectorName"] = sector
+        log(f"CF: sectorName -> {sector}")
+    if small and cf.get("smallCellName") != small:
+        cf_patch["smallCellName"] = small
+        log(f"CF: smallCellName -> {small}")
+
+    dev_patch = {}
+    if serial and (dev_full.get("serial") != serial):
+        dev_patch["serial"] = serial
+        log(f"DEV: serial -> {serial}")
+    if cf_patch:
+        dev_patch["custom_fields"] = cf_patch
+    nb.patch_device(dev_id, dev_patch)
+
+    # 4) IP handling (skip if placeholder)
+    ip_is_placeholder = (ip_from_cloud in (None, "", "0.0.0.0"))
+    ip_out_for_status = current_nb_ip  # default to current NB IP
+
+    if ip_is_placeholder:
+        log(f"IP: cloud reported placeholder '{ip_from_cloud}', skipping IP changes; keeping NetBox ip(s) as-is")
+    else:
+        iface_id = nb.ensure_eth0(dev_id)
+
+        ip_rec = nb.get_ip_by_addr(ip_from_cloud)
+        if ip_rec is None:
+            log(f"IP: create new {ip_from_cloud} on iface {iface_id}")
+            ip_id = nb.create_ip_for_iface(ip_from_cloud, iface_id)
+        else:
+            ip_id = ip_rec["id"]
+            assigned_type = ip_rec.get("assigned_object_type") or ""
+            assigned_id = ip_rec.get("assigned_object_id")
+            if not assigned_type:
+                if not nb.assign_ip_to_iface(ip_id, iface_id):
+                    die(4, f"FAIL assign IP {ip_from_cloud} to iface {iface_id}")
+            elif assigned_type == "dcim.interface":
+                if str(assigned_id) != str(iface_id):
+                    other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
+                    log(f"IP: moving {ip_from_cloud} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id}")
+                    # Clear old device primary if necessary
+                    if other_dev:
+                        r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
+                        if r.status_code == 200:
+                            old_primary_id = (r.json().get("primary_ip4") or {}).get("id")
+                            if str(old_primary_id) == str(ip_id):
+                                S_NB.patch(nb._url(f"/api/dcim/devices/{other_dev}/"),
+                                           headers=nb._h(), data=json.dumps({"primary_ip4": None}),
+                                           timeout=REQ_TIMEOUT)
+                    if not nb.assign_ip_to_iface(ip_id, iface_id):
+                        die(4, f"FAIL move IP {ip_from_cloud} to iface {iface_id}")
+            else:
+                die(4, f"FAIL IP {ip_from_cloud} assigned to {assigned_type}")
+
+        nb.device_set_primary_ip4(dev_id, ip_id)
+        ip_out_for_status = ip_from_cloud
+
+        # PRUNE all other IPs on this device (default behavior)
+        all_ips = nb.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get("id")
+            if str(rid) == str(ip_id):
+                continue
+            addr = rec.get("address")
+            log(f"IP: pruning stale {addr} (id={rid}) from device {dev_id}")
+            nb.delete_ip(rid)
+
+    print(f"OK {hostname} ip={ip_out_for_status or 'NONE'} fw={fw} node={node} sector={sector} small={small}")
+
+# ------------ CLI ------------
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname (only if online)")
+    ap.add_argument("hostname", help="Device name in NetBox")
+    ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
+    args = ap.parse_args()
+
+    CHATTY = bool(args.chatty)  # module-scope assignment
+    try:
+        run(args.hostname)
+    except requests.RequestException as e:
+        die(3, f"FAIL network error: {e}")

files/nb_onedevice_update.py

@@ -0,0 +1,415 @@
+#!/usr/bin/env python3
+"""
+nb_sync_one_device.py
+
+Update a single NetBox device from Cloud by hostname.
+
+Behavior
+- Liveness gate via /v1/devices/{cloud_id}/liveness.
+  * If online: keep ORIGINAL behavior (Cloud is SoT, including IP).
+  * If offline: query Subsystem for the device and use Subsystem IP as fallback,
+    then continue with normal NetBox updates.
+- Updates NetBox fields from Cloud:
+  custom_fields.fw_version ← firmwareVersion
+  custom_fields.nodeName   ← nodeName
+  custom_fields.sectorName ← sectorName
+  custom_fields.smallCellName ← smallCellName
+  serial ← serialNumber
+- IP handling:
+  If chosen IP (Cloud when online, Subsystem when offline) is valid (not None/""/"0.0.0.0"):
+    ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
+    then PRUNE all other IPs on this device (default).
+  If chosen IP is placeholder/invalid:
+    skip IP changes and do not prune.
+- --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
+- Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+
+Exit codes:
+  0 = success
+  1 = not found / missing data
+  3 = network/HTTP error
+  4 = NetBox update error
+"""
+
+import sys
+import json
+import argparse
+from typing import Optional, Union, List, Dict, Any
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# ------------ HARD-CODED CONFIG (per Pavel) ------------
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v1/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
+
+
+# NEW: Subsystem fallback (only used when Cloud liveness isConnected=false)
+SUBSYSTEM_BASE = "https://subsystem.ikeja.co.za"
+SUBSYSTEM_TOKEN = "HJL+&XRCoeHwh5?13@gxvg86qD#kQfgc"
+SUBSYSTEM_OUTDOOR_ENDPOINT = "/customers/wave-devices/get-outdoor-devices"
+
+REQ_TIMEOUT = 30
+CHATTY = False
+
+# ------------ HTTP utilities ------------
+def _new_session() -> requests.Session:
+    s = requests.Session()
+    retries = Retry(
+        total=3, connect=3, read=3, status=3,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+        respect_retry_after_header=True,
+    )
+    adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
+    s.mount("http://", adapter)
+    s.mount("https://", adapter)
+    s.headers["Accept"] = "application/json"
+    return s
+
+S_NB = _new_session()
+S_CL = _new_session()
+S_SUB = _new_session()
+
+# ------------ Logging / status helpers ------------
+def log(msg: str):
+    if CHATTY:
+        print(msg, file=sys.stderr)
+
+def die(code: int, msg: str):
+    print(msg, file=sys.stdout if code == 0 else sys.stderr)
+    raise SystemExit(code)
+
+def _is_placeholder_ip(v: Any) -> bool:
+    return v in (None, "", "0.0.0.0")
+
+def _clean_ip(v: Any) -> Any:
+    if isinstance(v, str):
+        return v.strip()
+    return v
+
+# ------------ NetBox API ------------
+class NetBox:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip("/")
+        self.token = token
+
+    def _h(self):
+        return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def get_device_by_name(self, name: str) -> Optional[dict]:
+        log(f"NB: lookup device by name {name}")
+        r = S_NB.get(self._url("/api/dcim/devices/"), headers=self._h(),
+                     params={"name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET devices name={name} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def get_device(self, dev_id: int) -> dict:
+        log(f"NB: fetch device id={dev_id}")
+        r = S_NB.get(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET device id={dev_id} HTTP={r.status_code}")
+        return r.json()
+
+    def patch_device(self, dev_id: int, patch: dict) -> None:
+        if not patch:
+            log("NB: no device patch needed")
+            return
+        log(f"NB: patch device id={dev_id} keys={list(patch.keys())}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps(patch), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL patch device dev={dev_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                     params={"device_id": dev_id, "name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET interfaces device_id={dev_id} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0]["id"] if res else None
+
+    def ensure_eth0(self, dev_id: int) -> int:
+        ifid = self.get_iface_id(dev_id, "eth0")
+        log(f"NB: ensure eth0 (current id={ifid})")
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = S_NB.post(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        if r.status_code == 400:
+            ifid = self.get_iface_id(dev_id, "eth0")
+            if ifid:
+                return ifid
+        die(4, f"FAIL create eth0 HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_ip_by_addr(self, addr: str) -> Optional[dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"address": f"{addr}/32"}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET ip-addresses addr={addr} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def create_ip_for_iface(self, addr: str, iface_id: int) -> int:
+        payload = {
+            "address": f"{addr}/32",
+            "status": "active",
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        }
+        r = S_NB.post(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        die(4, f"FAIL create IP {addr} HTTP={r.status_code} body={r.text[:200]}")
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
+        r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
+                       data=json.dumps({"assigned_object_type": "dcim.interface", "assigned_object_id": iface_id}),
+                       timeout=REQ_TIMEOUT)
+        return 200 <= r.status_code < 300
+
+    def device_set_primary_ip4(self, dev_id: int, ip_id: int) -> None:
+        log(f"NB: set primary_ip4 dev={dev_id} -> ip_id={ip_id}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps({"primary_ip4": ip_id}), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL set primary_ip4 dev={dev_id} ip_id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = S_NB.get(self._url(f"/api/dcim/interfaces/{iface_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return (r.json().get("device") or {}).get("id")
+        return None
+
+    def list_device_ips(self, dev_id: int) -> List[Dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return r.json().get("results") or []
+        return []
+
+    def delete_ip(self, ip_id: int) -> None:
+        log(f"NB: delete IP id={ip_id}")
+        r = S_NB.delete(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300 or r.status_code == 204):
+            die(4, f"FAIL delete IP id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+# ------------ Cloud API ------------
+class Cloud:
+    def __init__(self, base: str, bearer: str):
+        self.base = base.rstrip("/")
+        self.bearer = bearer
+
+    def _h(self):
+        return {"Authorization": f"Bearer {self.bearer}", "Accept": "application/json"}
+
+    def device_detail(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}"
+        log(f"CL: fetch detail cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id} HTTP={r.status_code}")
+        return r.json()
+
+    def device_liveness(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}/liveness"
+        log(f"CL: fetch liveness cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
+        return r.json()
+
+# ------------ Subsystem fallback ------------
+class Subsystem:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip("/")
+        self.token = token
+
+    def _h(self):
+        return {"token": self.token, "Accept": "application/json"}
+
+    def fetch_outdoor_devices(self) -> List[Dict[str, Any]]:
+        url = f"{self.base}{SUBSYSTEM_OUTDOOR_ENDPOINT}"
+        log("SUBSYS: fetch outdoor devices list")
+        r = S_SUB.post(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Subsystem POST get-outdoor-devices HTTP={r.status_code}")
+        data = r.json()
+        return data.get("device_list") or []
+
+    def find_by_name(self, name: str) -> Optional[Dict[str, Any]]:
+        devs = self.fetch_outdoor_devices()
+        want = name.lower()
+        for rec in devs:
+            n = rec.get("name")
+            if isinstance(n, str) and n.lower() == want:
+                return rec
+        return None
+
+# ------------ Core ------------
+def _primary_ip4_text(dev_json: dict) -> Optional[str]:
+    p = dev_json.get("primary_ip4") or {}
+    addr = p.get("address")
+    if isinstance(addr, str) and addr.endswith("/32"):
+        return addr[:-3]
+    return addr
+
+def run(hostname: str) -> None:
+    nb = NetBox(NB_URL, NB_TOKEN)
+    cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+    sub = Subsystem(SUBSYSTEM_BASE, SUBSYSTEM_TOKEN)
+
+    dev = nb.get_device_by_name(hostname)
+    if not dev:
+        die(1, f"FAIL {hostname} not found in NetBox")
+
+    dev_id = dev["id"]
+    dev_full = nb.get_device(dev_id)
+    cf = dev_full.get("custom_fields") or {}
+
+    upgrade_cmd = cf.get("upgrade_cmd")
+    log(f"NB: upgrade_cmd={upgrade_cmd}")
+
+    cloud_id = cf.get("cloud_id")
+    if cloud_id in (None, "", "null"):
+        die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
+
+    # 1) Cloud liveness gate (ONLINE path stays as-is)
+    live = cl.device_liveness(cloud_id)
+    cloud_connected = bool(live.get("isConnected"))
+    if cloud_connected:
+        log(f"CL: liveness isConnected=true for cloud_id={cloud_id}; using Cloud ipAddress as source of truth")
+    else:
+        log(f"CL: liveness isConnected=false for cloud_id={cloud_id}; falling back to Subsystem for IP")
+
+    current_nb_ip = _primary_ip4_text(dev_full)
+
+    # 2) Cloud detail (still used for fw/node/sector/small/serial)
+    d = cl.device_detail(cloud_id)
+    fw = d.get("firmwareVersion") or d.get("version")
+    cloud_ip = _clean_ip(d.get("ipAddress"))
+    node = d.get("nodeName")
+    sector = d.get("sectorName")
+    small = d.get("smallCellName")
+    serial = d.get("serialNumber")
+
+    if not fw:
+        die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
+
+    # Choose IP source
+    chosen_ip = cloud_ip
+    ip_source = "cloud"
+
+    if not cloud_connected:
+        sub_rec = sub.find_by_name(hostname)
+        if not sub_rec:
+            # No subsystem record -> keep previous behavior: fail because offline and no fallback source
+            die(1, f"FAIL {hostname} cloud_id={cloud_id} device offline and not found in Subsystem")
+        sub_ip = _clean_ip(sub_rec.get("ip"))
+        chosen_ip = sub_ip
+        ip_source = "subsystem"
+        log(f"SUBSYS: {hostname} ip={sub_ip} (fallback)")
+
+    # 3) NetBox patch (idempotent)
+    cf_patch = {}
+    if cf.get("fw_version") != fw:
+        cf_patch["fw_version"] = fw
+        log(f"CF: fw_version -> {fw}")
+    if node and cf.get("nodeName") != node:
+        cf_patch["nodeName"] = node
+        log(f"CF: nodeName -> {node}")
+    if sector and cf.get("sectorName") != sector:
+        cf_patch["sectorName"] = sector
+        log(f"CF: sectorName -> {sector}")
+    if small and cf.get("smallCellName") != small:
+        cf_patch["smallCellName"] = small
+        log(f"CF: smallCellName -> {small}")
+
+    dev_patch = {}
+    if serial and (dev_full.get("serial") != serial):
+        dev_patch["serial"] = serial
+        log(f"DEV: serial -> {serial}")
+    if cf_patch:
+        dev_patch["custom_fields"] = cf_patch
+    nb.patch_device(dev_id, dev_patch)
+
+    # 4) IP handling (skip if placeholder)
+    chosen_ip = _clean_ip(chosen_ip)
+    ip_is_placeholder = _is_placeholder_ip(chosen_ip)
+    ip_out_for_status = current_nb_ip  # default to current NB IP
+
+    if ip_is_placeholder:
+        log(f"IP: {ip_source} reported placeholder '{chosen_ip}', skipping IP changes; keeping NetBox ip(s) as-is")
+    else:
+        iface_id = nb.ensure_eth0(dev_id)
+
+        ip_rec = nb.get_ip_by_addr(chosen_ip)
+        if ip_rec is None:
+            log(f"IP: create new {chosen_ip} on iface {iface_id} (source={ip_source})")
+            ip_id = nb.create_ip_for_iface(chosen_ip, iface_id)
+        else:
+            ip_id = ip_rec["id"]
+            assigned_type = ip_rec.get("assigned_object_type") or ""
+            assigned_id = ip_rec.get("assigned_object_id")
+            if not assigned_type:
+                if not nb.assign_ip_to_iface(ip_id, iface_id):
+                    die(4, f"FAIL assign IP {chosen_ip} to iface {iface_id}")
+            elif assigned_type == "dcim.interface":
+                if str(assigned_id) != str(iface_id):
+                    other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
+                    log(f"IP: moving {chosen_ip} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id} (source={ip_source})")
+                    # Clear old device primary if necessary
+                    if other_dev:
+                        r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
+                        if r.status_code == 200:
+                            old_primary_id = (r.json().get("primary_ip4") or {}).get("id")
+                            if str(old_primary_id) == str(ip_id):
+                                S_NB.patch(nb._url(f"/api/dcim/devices/{other_dev}/"),
+                                           headers=nb._h(), data=json.dumps({"primary_ip4": None}),
+                                           timeout=REQ_TIMEOUT)
+                    if not nb.assign_ip_to_iface(ip_id, iface_id):
+                        die(4, f"FAIL move IP {chosen_ip} to iface {iface_id}")
+            else:
+                die(4, f"FAIL IP {chosen_ip} assigned to {assigned_type}")
+
+        nb.device_set_primary_ip4(dev_id, ip_id)
+        ip_out_for_status = chosen_ip
+
+        # PRUNE all other IPs on this device (default behavior)
+        for rec in nb.list_device_ips(dev_id):
+            rid = rec.get("id")
+            if str(rid) == str(ip_id):
+                continue
+            addr = rec.get("address")
+            log(f"IP: pruning stale {addr} (id={rid}) from device {dev_id}")
+            nb.delete_ip(rid)
+
+    print(f"OK {hostname} ip={ip_out_for_status or 'NONE'} fw={fw} node={node} sector={sector} small={small}")
+
+# ------------ CLI ------------
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname")
+    ap.add_argument("hostname", help="Device name in NetBox")
+    ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
+    args = ap.parse_args()
+
+    CHATTY = bool(args.chatty)
+    try:
+        run(args.hostname)
+    except requests.RequestException as e:
+        die(3, f"FAIL network error: {e}")

files/netbox_subsystem_ikejanum_recursive.py

@@ -0,0 +1,885 @@
+#!/usr/bin/env python3
+"""
+Recursive single-target sync: Subsystem -> NetBox (Source of Truth), with depth.
+
+Usage:
+  ./netbox_subsystem_ikejanum <device_name> [--deep N]
+
+Behavior:
+- Start with <device_name>.
+- Sync from Subsystem into NetBox (create device if missing).
+- If device missing in Subsystem but present in NetBox -> set NetBox status=failed.
+- While syncing:
+    * If an IP is MOVED from another NetBox device -> enqueue that old device for sync (depth+1)
+    * If a stale IP is being DELETED from a device -> find Subsystem device that owns that IP and enqueue it (depth+1)
+
+Depth:
+- Default depth: 3
+- --deep N allowed, but N is hard-limited to <= 10.
+
+NEW:
+- --info-only: Print the raw Subsystem record (JSON) for the device and exit.
+  No NetBox changes, no recursion.
+
+Exit codes:
+  0 success
+  1 runtime error / not found (info-only)
+  2 config/arg error
+"""
+
+# =========================
+# USER CONFIG — EDIT HERE
+# =========================
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+
+SUBSYSTEM_BASE = "https://subsystem.ikeja.co.za"
+SUBSYSTEM_TOKEN = "HJL+&XRCoeHwh5?13@gxvg86qD#kQfgc"
+
+ROLE_CPE = 1
+TYPE_FOX100_CPE = 1
+TYPE_FOX200 = 2
+SITE_ID = 1
+
+DEFAULT_MIN_LAST_DETECTED_MINUTES = 0  # 0 = accept any age
+DEFAULT_DEEP = 3
+MAX_DEEP = 10
+# =========================
+
+import os
+import sys
+import json
+import time
+import logging
+import argparse
+from typing import Optional, Dict, Any, Tuple, List, Set, Deque
+from collections import deque
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+
+# ------------- Logging -------------
+def setup_logging(troubleshoot: bool) -> None:
+    level = logging.DEBUG if troubleshoot else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format='%(asctime)s %(levelname)s %(message)s',
+        datefmt='%H:%M:%S'
+    )
+
+def summarize_body(body: Any, limit: int = 500) -> str:
+    if body is None:
+        return "<none>"
+    if isinstance(body, (dict, list)):
+        s = json.dumps(body)
+    else:
+        s = str(body)
+    if len(s) > limit:
+        return s[:limit] + f"... (+{len(s)-limit}B)"
+    return s
+
+def is_ascii_or_die(label: str, value: str) -> None:
+    try:
+        value.encode('latin-1')
+    except UnicodeEncodeError:
+        bad = ''.join(ch for ch in value if ord(ch) > 127)
+        logging.error(
+            "%s contains non-ASCII characters (e.g. %r). Please paste the exact token without smart punctuation.",
+            label, bad,
+        )
+        raise SystemExit(2)
+
+# ------------- HTTP Session helpers -------------
+class Http:
+    def __init__(self, troubleshoot: bool = False):
+        self.s_nb = self._new_session()
+        self.s_subsystem = self._new_session()
+        self.troubleshoot = troubleshoot
+
+    def _new_session(self) -> requests.Session:
+        s = requests.Session()
+        retries = Retry(
+            total=3,
+            connect=3,
+            read=3,
+            status=3,
+            backoff_factor=0.75,
+            status_forcelist=(429, 500, 502, 503, 504),
+            allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+            respect_retry_after_header=True,
+        )
+        adapter = HTTPAdapter(max_retries=retries, pool_connections=16, pool_maxsize=32)
+        s.mount('http://', adapter)
+        s.mount('https://', adapter)
+        s.headers['Accept'] = 'application/json'
+        s.headers['Content-Type'] = 'application/json'
+        return s
+
+    def nb(self, method: str, url: str, token: str, **kw) -> requests.Response:
+        headers = kw.pop('headers', {})
+        headers['Authorization'] = f'Token {token}'
+        t0 = time.time()
+        resp = self.s_nb.request(method, url, headers=headers, timeout=30, **kw)
+        dt = (time.time() - t0) * 1000
+        if self.troubleshoot or resp.status_code >= 400:
+            logging.debug(
+                "NB %s %s [%s] %.1fms\n  req:%s\n  resp:%s",
+                method, url, resp.status_code, dt,
+                summarize_body(kw.get('data') or kw.get('json')),
+                summarize_body(resp.text),
+            )
+        return resp
+
+    def subsystem(self, method: str, url: str, token: str, **kw) -> requests.Response:
+        headers = kw.pop('headers', {})
+        headers['token'] = token
+        t0 = time.time()
+        resp = self.s_subsystem.request(method, url, headers=headers, timeout=30, **kw)
+        dt = (time.time() - t0) * 1000
+        if self.troubleshoot or resp.status_code >= 400:
+            logging.debug(
+                "SUBSYS %s %s [%s] %.1fms\n  resp:%s",
+                method, url, resp.status_code, dt,
+                summarize_body(resp.text),
+            )
+        return resp
+
+# ------------- Utility -------------
+def normalize_mac_from_subsystem(mac_raw: str) -> Optional[str]:
+    if not mac_raw:
+        return None
+    s = mac_raw.strip().replace(":", "").replace("-", "")
+    s = s.upper()
+    try:
+        chunks = [s[i:i+2] for i in range(0, len(s), 2)]
+        return ":".join(chunks)
+    except Exception:
+        logging.warning("  !! failed to normalize MAC from subsystem: %r", mac_raw)
+        return None
+
+def parse_last_detected(ts: Optional[str]) -> Optional[datetime]:
+    if not ts:
+        return None
+    try:
+        dt = parsedate_to_datetime(ts)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        else:
+            dt = dt.astimezone(timezone.utc)
+        return dt
+    except Exception:
+        logging.warning("  !! cannot parse last_detected: %r", ts)
+        return None
+
+def minutes_age(now_utc: datetime, past_utc: datetime) -> float:
+    return (now_utc - past_utc).total_seconds() / 60.0
+
+def ip_strip_prefix(ip_with_prefix: str) -> str:
+    # "10.0.0.1/32" -> "10.0.0.1"
+    return (ip_with_prefix or "").split("/")[0].strip()
+
+# ------------- Subsystem client (with caching) -------------
+class Subsystem:
+    def __init__(self, base: str, token: str, http: Http):
+        self.base = base.rstrip('/')
+        self.token = token
+        self.http = http
+        self._cache_devices: Optional[list[Dict[str, Any]]] = None
+
+    def fetch_outdoor_devices(self) -> list[Dict[str, Any]]:
+        url = f"{self.base}/customers/wave-devices/get-outdoor-devices"
+        r = self.http.subsystem("POST", url, self.token)
+        r.raise_for_status()
+        data = r.json()
+        return data.get("device_list") or []
+
+    def _ensure_cache(self) -> None:
+        if self._cache_devices is None:
+            self._cache_devices = self.fetch_outdoor_devices()
+
+    def find_device_by_name(self, target_name: str) -> Optional[Dict[str, Any]]:
+        self._ensure_cache()
+        assert self._cache_devices is not None
+        for rec in self._cache_devices:
+            name = (rec.get("name") or "").strip()
+            if name.lower() == target_name.lower():
+                return rec
+        return None
+
+    def find_device_by_ip(self, ip: str) -> Optional[Dict[str, Any]]:
+        if not ip:
+            return None
+        self._ensure_cache()
+        assert self._cache_devices is not None
+        ip = ip.strip()
+        for rec in self._cache_devices:
+            rip = (rec.get("ip") or "").strip()
+            if rip == ip:
+                return rec
+        return None
+
+# ------------- NetBox API wrappers -------------
+class NetBox:
+    def __init__(
+        self,
+        base: str,
+        token: str,
+        role_cpe: int,
+        type_fox100: int,
+        type_fox200: int,
+        site_id: int,
+        http: Http,
+        dry_run: bool = False,
+    ):
+        self.base = base.rstrip('/')
+        self.token = token
+        self.role_cpe = role_cpe
+        self.type_fox100 = type_fox100
+        self.type_fox200 = type_fox200
+        self.site_id = site_id
+        self.http = http
+        self.dry = dry_run
+        self.allow_moves = True
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def _req(self, method: str, path: str, **kw) -> requests.Response:
+        if self.dry and method in ("POST", "PATCH", "PUT", "DELETE"):
+            logging.info("DRY %s %s", method, path)
+            r = requests.Response()
+            if method == "POST":
+                r.status_code = 201
+                r._content = b'{"id": 0}'
+            else:
+                r.status_code = 200
+                r._content = b'{}'
+            r.headers['Content-Type'] = 'application/json'
+            return r
+        return self.http.nb(method, self._url(path), self.token, **kw)
+
+    # --- devices ---
+    def get_device_id_by_name(self, name: str) -> Optional[int]:
+        r = self._req("GET", "/api/dcim/devices/", params={"name": name})
+        r.raise_for_status()
+        data = r.json()
+        if data.get('results'):
+            return data['results'][0]['id']
+        return None
+
+    def get_device(self, dev_id: int) -> Dict[str, Any]:
+        r = self._req("GET", f"/api/dcim/devices/{dev_id}/")
+        r.raise_for_status()
+        return r.json()
+
+    def get_device_name(self, dev_id: int) -> Optional[str]:
+        r = self._req("GET", f"/api/dcim/devices/{dev_id}/")
+        if r.status_code == 200:
+            return r.json().get("name")
+        return None
+
+    def create_device(self, name: str, dtype_id: int) -> Optional[int]:
+        payload = {
+            "name": name,
+            "role": self.role_cpe,
+            "device_type": dtype_id,
+            "site": self.site_id,
+            "status": "active",
+        }
+        r = self._req("POST", "/api/dcim/devices/", json=payload)
+        if r.status_code == 201:
+            dev_id = r.json().get('id')
+            logging.info("  -> created NetBox device name=%s id=%s", name, dev_id)
+            return dev_id
+        logging.error("device create failed (%s) HTTP=%s body=%s", name, r.status_code, summarize_body(r.text))
+        return None
+
+    def patch_device(self, dev_id: int, patch: Dict[str, Any]) -> bool:
+        if not patch:
+            return True
+        r = self._req("PATCH", f"/api/dcim/devices/{dev_id}/", json=patch)
+        if 200 <= r.status_code < 300:
+            return True
+        logging.error("device patch failed dev=%s HTTP=%s body=%s", dev_id, r.status_code, summarize_body(r.text))
+        return False
+
+    def set_primary_ip4_if_changed(self, dev: Dict[str, Any], ip_id: int) -> bool:
+        curr = (dev.get('primary_ip4') or {}).get('id')
+        if curr == ip_id:
+            return True
+        r = self._req("PATCH", f"/api/dcim/devices/{dev['id']}/", json={"primary_ip4": ip_id})
+        ok = 200 <= r.status_code < 300
+        if not ok:
+            logging.error("set primary_ip4 failed dev=%s HTTP=%s body=%s", dev['id'], r.status_code, summarize_body(r.text))
+        return ok
+
+    def set_status(self, dev_id: int, status: str) -> bool:
+        r = self._req("PATCH", f"/api/dcim/devices/{dev_id}/", json={"status": status})
+        if 200 <= r.status_code < 300:
+            return True
+        logging.error("status patch failed dev=%s status=%s HTTP=%s body=%s", dev_id, status, r.status_code, summarize_body(r.text))
+        return False
+
+    def add_journal_entry(self, dev_id: int, comments: str, kind: str = "info") -> bool:
+        payload = {
+            "assigned_object_type": "dcim.device",
+            "assigned_object_id": dev_id,
+            "kind": kind,
+            "comments": comments,
+        }
+        r = self._req("POST", "/api/extras/journal-entries/", json=payload)
+        if 200 <= r.status_code < 300:
+            return True
+        logging.error(
+            "journal entry create failed dev=%s kind=%s HTTP=%s body=%s",
+            dev_id, kind, r.status_code, summarize_body(r.text)
+        )
+        return False
+
+    # --- interfaces ---
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = self._req("GET", "/api/dcim/interfaces/", params={"device_id": dev_id, "name": name})
+        r.raise_for_status()
+        data = r.json()
+        if data.get('results'):
+            return data['results'][0]['id']
+        return None
+
+    def ensure_eth0(self, dev_id: int) -> Optional[int]:
+        ifid = self.get_iface_id(dev_id, 'eth0')
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = self._req("POST", "/api/dcim/interfaces/", json=payload)
+        if r.status_code == 201:
+            return r.json()['id']
+        if r.status_code == 400:
+            return self.get_iface_id(dev_id, 'eth0')
+        logging.error("interface create failed device=%s HTTP=%s body=%s", dev_id, r.status_code, summarize_body(r.text))
+        return None
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = self._req("GET", f"/api/dcim/interfaces/{iface_id}/")
+        if r.status_code == 200:
+            return (r.json().get('device') or {}).get('id')
+        return None
+
+    def get_interface_detail(self, iface_id: int) -> Optional[Dict[str, Any]]:
+        r = self._req("GET", f"/api/dcim/interfaces/{iface_id}/")
+        if 200 <= r.status_code < 300:
+            return r.json()
+        logging.error("interface fetch failed iface=%s HTTP=%s body=%s", iface_id, r.status_code, summarize_body(r.text))
+        return None
+
+    # --- IP inventory helpers ---
+    def list_device_ips(self, dev_id: int) -> list[Dict[str, Any]]:
+        r = self._req("GET", "/api/ipam/ip-addresses/", params={"device_id": dev_id, "limit": 1000})
+        if r.status_code == 200:
+            return r.json().get('results') or []
+        # fallback by interface
+        ips: list[Dict[str, Any]] = []
+        r2 = self._req("GET", "/api/dcim/interfaces/", params={"device_id": dev_id, "limit": 1000})
+        if r2.status_code == 200:
+            for iface in (r2.json().get('results') or []):
+                ifid = iface.get('id')
+                r3 = self._req("GET", "/api/ipam/ip-addresses/", params={
+                    "assigned_object_type": "dcim.interface",
+                    "assigned_object_id": ifid,
+                    "limit": 1000,
+                })
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get('results') or [])
+        return ips
+
+    def prune_other_ips_with_details(self, dev_id: int, keep_ip_id: int) -> List[Tuple[int, str]]:
+        """
+        Delete all other IPs assigned to this device, keeping only keep_ip_id.
+        Returns list of (ip_id, address) that were removed.
+        """
+        removed: List[Tuple[int, str]] = []
+        all_ips = self.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get('id')
+            if rid == keep_ip_id:
+                continue
+            addr = rec.get('address')
+            logging.warning("  -> removing stale IP %s (id=%s) from device %s", addr, rid, dev_id)
+            self._req("DELETE", f"/api/ipam/ip-addresses/{rid}/")
+            if rid is not None and addr:
+                removed.append((int(rid), str(addr)))
+        return removed
+
+    # --- MAC helpers ---
+    def ensure_single_mac_on_iface(self, iface_id: int, mac_norm: str) -> Tuple[int, int]:
+        created = 0
+        deleted = 0
+        iface = self.get_interface_detail(iface_id)
+        if not iface:
+            logging.error("  !! cannot fetch interface detail for MAC sync (iface=%s)", iface_id)
+            return (0, 0)
+
+        macs = iface.get("mac_addresses") or []
+        matching_ids: list[int] = []
+        bad_ids: list[int] = []
+
+        for m in macs:
+            mid = m.get("id")
+            mval = (m.get("mac_address") or "").upper()
+            if mval == mac_norm:
+                if mid is not None:
+                    matching_ids.append(mid)
+            else:
+                if mid is not None:
+                    bad_ids.append(mid)
+
+        if not matching_ids:
+            payload = {
+                "mac_address": mac_norm,
+                "assigned_object_type": "dcim.interface",
+                "assigned_object_id": iface_id,
+            }
+            logging.info("  -> create MAC %s on iface %s", mac_norm, iface_id)
+            r = self._req("POST", "/api/dcim/mac-addresses/", json=payload)
+            if r.status_code == 201:
+                created += 1
+            else:
+                logging.error("  !! MAC create failed iface=%s mac=%s HTTP=%s body=%s", iface_id, mac_norm, r.status_code, summarize_body(r.text))
+        else:
+            extra_ids = matching_ids[1:]
+            bad_ids.extend([i for i in extra_ids if i is not None])
+
+        for mid in bad_ids:
+            logging.warning("  -> removing stale/duplicate MAC entry id=%s from iface=%s", mid, iface_id)
+            self._req("DELETE", f"/api/dcim/mac-addresses/{mid}/")
+            deleted += 1
+
+        return (created, deleted)
+
+    # --- IP addresses ---
+    def get_ip_by_address(self, addr: str) -> Dict[str, Any] | None:
+        r = self._req("GET", "/api/ipam/ip-addresses/", params={"address": f"{addr}/32"})
+        r.raise_for_status()
+        res = r.json().get('results') or []
+        return res[0] if res else None
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        r = self._req("PATCH", f"/api/ipam/ip-addresses/{ip_id}/", json={
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        })
+        return r.status_code == 200
+
+    def ensure_ip_for_device(self, dev_id: int, iface_id: int, addr: str) -> Tuple[Optional[int], str, Optional[int]]:
+        """
+        Ensure addr/32 exists and is assigned to iface_id on dev_id.
+        Returns (ip_id, action, old_device_id_if_moved)
+          action: created, reused, assigned, moved, skipped, error
+        """
+        ip_rec = self.get_ip_by_address(addr)
+        if not ip_rec:
+            payload = {
+                "address": f"{addr}/32",
+                "status": "active",
+                "assigned_object_type": "dcim.interface",
+                "assigned_object_id": iface_id,
+            }
+            r = self._req("POST", "/api/ipam/ip-addresses/", json=payload)
+            if r.status_code == 201:
+                return (r.json()['id'], "created", None)
+            logging.error("ip create failed iface=%s addr=%s HTTP=%s body=%s", iface_id, addr, r.status_code, summarize_body(r.text))
+            return (None, "error", None)
+
+        ip_id = ip_rec['id']
+        aot = ip_rec.get('assigned_object_type') or ''
+        if not aot:
+            ok = self.assign_ip_to_iface(ip_id, iface_id)
+            return (ip_id if ok else None, "assigned" if ok else "error", None)
+
+        if aot == 'dcim.interface':
+            assigned_ifid = ip_rec.get('assigned_object_id')
+            if str(assigned_ifid) == str(iface_id):
+                return (ip_id, "reused", None)
+
+            assigned_dev = self.get_device_id_of_interface(assigned_ifid) if assigned_ifid else None
+            if assigned_dev and str(assigned_dev) == str(dev_id):
+                logging.info("  -> IP %s already on same device (iface=%s); reusing.", addr, assigned_ifid)
+                return (ip_id, "reused", None)
+
+            if self.allow_moves:
+                old_dev_id = assigned_dev
+
+                # clear primary_ip4 on old device if needed
+                if old_dev_id:
+                    r_old = self._req("GET", f"/api/dcim/devices/{old_dev_id}/")
+                    if r_old.status_code == 200:
+                        old_primary_id = (r_old.json().get("primary_ip4") or {}).get("id")
+                        if str(old_primary_id) == str(ip_id):
+                            r_clr = self._req("PATCH", f"/api/dcim/devices/{old_dev_id}/", json={"primary_ip4": None})
+                            if not (200 <= r_clr.status_code < 300):
+                                logging.error("  -> cannot clear old device %s primary_ip4 for IP %s; aborting move", old_dev_id, addr)
+                                return (None, "error", None)
+
+                logging.warning(
+                    "  -> IP %s currently belongs to device %s (iface %s); moving to this device (iface %s)",
+                    addr, old_dev_id, assigned_ifid, iface_id,
+                )
+                r2 = self._req("PATCH", f"/api/ipam/ip-addresses/{ip_id}/", json={
+                    "assigned_object_type": "dcim.interface",
+                    "assigned_object_id": iface_id,
+                })
+                if r2.status_code == 200:
+                    return (ip_id, "moved", old_dev_id)
+                logging.error("  -> move failed for IP %s HTTP=%s body=%s", addr, r2.status_code, summarize_body(r2.text))
+                return (None, "error", None)
+
+            return (None, "skipped", None)
+
+        logging.warning("  -> IP %s assigned to %s; skipping.", addr, aot)
+        return (None, "skipped", None)
+
+# ------------- Core sync functions -------------
+def sync_from_subsystem_record(
+    target: str,
+    rec: Dict[str, Any],
+    nb: NetBox,
+    subsystem: Subsystem,
+    now_utc: datetime,
+    min_last_detected_minutes: int,
+    enqueue_fn,
+) -> Dict[str, Any]:
+    """
+    enqueue_fn(name: str, reason: str) -> None
+    """
+    summary = {
+        "target": target,
+        "subsystem_found": True,
+        "netbox_created": False,
+        "status_set_active": False,
+        "node_updated": False,
+        "mac_created": 0,
+        "mac_deleted": 0,
+        "ip_action": "none",
+        "ips_pruned": 0,
+        "primary_set": False,
+        "skipped_stale": False,
+        "errors": 0,
+    }
+
+    name = rec.get("name") or target
+    ip = rec.get("ip")
+    mac_raw = rec.get("mac")
+    connected_node = rec.get("connected_node")
+    last_detected = rec.get("last_detected")
+
+    dt_last = parse_last_detected(last_detected)
+    if min_last_detected_minutes > 0:
+        if dt_last is None:
+            logging.info("  -> skip: last_detected unavailable; requires <= %s minutes", min_last_detected_minutes)
+            summary["skipped_stale"] = True
+            return summary
+        age_min = minutes_age(now_utc, dt_last)
+        if age_min > min_last_detected_minutes:
+            logging.info("  -> skip: last_detected age %.1f min > allowed %s min", age_min, min_last_detected_minutes)
+            summary["skipped_stale"] = True
+            return summary
+
+    dev_id = nb.get_device_id_by_name(name)
+    if not dev_id:
+        logging.info("  -> device not found in NetBox; creating")
+        dev_id = nb.create_device(name, nb.type_fox100)
+        if not dev_id:
+            summary["errors"] += 1
+            return summary
+        summary["netbox_created"] = True
+
+    dev = nb.get_device(dev_id)
+    curr_status = dev.get("status", {}).get("value") if isinstance(dev.get("status"), dict) else dev.get("status")
+    if curr_status != "active":
+        active_reason = "device present in subsystem"
+        if min_last_detected_minutes > 0:
+            active_reason += f" and last_detected within {min_last_detected_minutes} minutes"
+        else:
+            active_reason += " and freshness filter accepts it"
+        if nb.set_status(dev_id, "active"):
+            nb.add_journal_entry(dev_id, f"setting to active because {active_reason}", kind="info")
+            summary["status_set_active"] = True
+            dev = nb.get_device(dev_id)
+        else:
+            summary["errors"] += 1
+
+    cf = dev.get("custom_fields") or {}
+    current_node = (cf.get("nodeName") if isinstance(cf, dict) else None)
+
+    patch: Dict[str, Any] = {}
+    if connected_node and connected_node != current_node:
+        patch.setdefault("custom_fields", {})["nodeName"] = connected_node
+        logging.info("  -> patch custom_fields.nodeName: %r -> %r", current_node, connected_node)
+        if nb.patch_device(dev_id, patch):
+            summary["node_updated"] = True
+            dev = nb.get_device(dev_id)
+
+    iface_id = nb.ensure_eth0(dev_id)
+    if not iface_id:
+        logging.error("  !! interface ensure/create failed for device=%s", dev_id)
+        summary["errors"] += 1
+        return summary
+
+    mac_norm = normalize_mac_from_subsystem(mac_raw) if mac_raw else None
+    if mac_norm:
+        logging.info("  -> ensure single MAC %s on iface %s", mac_norm, iface_id)
+        c, d = nb.ensure_single_mac_on_iface(iface_id, mac_norm)
+        summary["mac_created"] += c
+        summary["mac_deleted"] += d
+    else:
+        logging.info("  -> MAC missing/unusable from subsystem; skipping MAC sync")
+
+    if ip and str(ip).lower() != "null" and ip != "0.0.0.0":
+        logging.info("  -> ensure IP %s for device %s (iface %s)", ip, dev_id, iface_id)
+        ip_id, ip_action, old_dev_id = nb.ensure_ip_for_device(dev_id, iface_id, ip)
+        summary["ip_action"] = ip_action
+
+        # If we moved from some other NetBox device, enqueue it for repair
+        if ip_action == "moved" and old_dev_id:
+            old_name = nb.get_device_name(old_dev_id)
+            if old_name and old_name.lower() != name.lower():
+                enqueue_fn(old_name, f"ip_moved_away:{ip}")
+
+        if ip_id:
+            removed = nb.prune_other_ips_with_details(dev_id, ip_id)
+            summary["ips_pruned"] = len(removed)
+
+            # For each deleted stale IP, see who owns that IP in Subsystem and enqueue them.
+            for _rid, addr_pref in removed:
+                stale_ip = ip_strip_prefix(addr_pref)
+                rec2 = subsystem.find_device_by_ip(stale_ip)
+                if rec2:
+                    n2 = rec2.get("name")
+                    if n2 and n2.lower() != name.lower():
+                        enqueue_fn(n2, f"stale_ip_deleted:{stale_ip}")
+
+            dev = nb.get_device(dev_id)
+            before = (dev.get("primary_ip4") or {}).get("id")
+            if nb.set_primary_ip4_if_changed(dev, ip_id):
+                if str(before) != str(ip_id):
+                    summary["primary_set"] = True
+        else:
+            summary["errors"] += 1
+    else:
+        logging.info("  -> IP missing or 0.0.0.0; skipping IP sync")
+
+    return summary
+
+def mark_failed_if_in_netbox_only(target: str, nb: NetBox) -> Dict[str, Any]:
+    summary = {
+        "target": target,
+        "subsystem_found": False,
+        "netbox_exists": False,
+        "status_set_failed": False,
+        "errors": 0,
+    }
+    dev_id = nb.get_device_id_by_name(target)
+    if not dev_id:
+        return summary
+    summary["netbox_exists"] = True
+
+    dev = nb.get_device(dev_id)
+    curr_status = dev.get("status", {}).get("value") if isinstance(dev.get("status"), dict) else dev.get("status")
+    if curr_status == "failed":
+        return summary
+
+    failed_reason = "device missing in subsystem"
+    if nb.set_status(dev_id, "failed"):
+        nb.add_journal_entry(dev_id, f"setting to failed, because {failed_reason}", kind="warning")
+        summary["status_set_failed"] = True
+    else:
+        summary["errors"] += 1
+    return summary
+
+def one_liner_single(result: Dict[str, Any]) -> str:
+    t = result.get("target", "?")
+    if result.get("subsystem_found") is False:
+        if result.get("netbox_exists"):
+            if result.get("status_set_failed"):
+                return f"{t}: not in subsystem -> NetBox status set to failed"
+            return f"{t}: not in subsystem -> NetBox status NOT changed (error)"
+        return f"{t}: not in subsystem and not in NetBox -> nothing to do"
+
+    if result.get("skipped_stale"):
+        return f"{t}: subsystem record skipped due to last_detected freshness filter"
+
+    parts = []
+    if result.get("netbox_created"):
+        parts.append("created in NetBox")
+    if result.get("status_set_active"):
+        parts.append("status set to active")
+    if result.get("node_updated"):
+        parts.append("nodeName updated")
+    if (result.get("mac_created", 0) or 0) > 0 or (result.get("mac_deleted", 0) or 0) > 0:
+        parts.append(f"mac c{result.get('mac_created',0)}/d{result.get('mac_deleted',0)}")
+    ia = result.get("ip_action")
+    if ia and ia != "none":
+        parts.append(f"ip {ia}")
+    if (result.get("ips_pruned", 0) or 0) > 0:
+        parts.append(f"ips pruned={result.get('ips_pruned')}")
+    if result.get("primary_set"):
+        parts.append("primary_ip4 set")
+    if (result.get("errors") or 0) > 0:
+        parts.append(f"errors={result.get('errors')}")
+
+    if not parts:
+        return f"{t}: already in sync (no changes)"
+    return f"{t}: " + ", ".join(parts)
+
+# ------------- Depth driver -------------
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("device_name", help="Exact device name to sync (e.g. ikeja12345)")
+    ap.add_argument(
+        "--info-only",
+        action="store_true",
+        default=False,
+        help="Print Subsystem record for the device and exit (no NetBox changes)",
+    )
+    ap.add_argument(
+        "--min-last-detected-minutes",
+        type=int,
+        default=DEFAULT_MIN_LAST_DETECTED_MINUTES,
+        help="Require subsystem last_detected within N minutes (0 = accept any age)",
+    )
+    ap.add_argument(
+        "--deep",
+        type=int,
+        default=DEFAULT_DEEP,
+        help=f"Recursive depth (default {DEFAULT_DEEP}, max {MAX_DEEP})",
+    )
+    ap.add_argument("--troubleshoot", action='store_true', default=bool(os.getenv('TROUBLESHOOT')))
+    ap.add_argument("--dry-run", action='store_true', default=False)
+    args = ap.parse_args()
+
+    setup_logging(args.troubleshoot)
+
+    if not NB_TOKEN:
+        logging.error("Set NB_TOKEN in script")
+        return 2
+    if not SUBSYSTEM_TOKEN:
+        logging.error("Set SUBSYSTEM_TOKEN in script")
+        return 2
+
+    is_ascii_or_die("NB_TOKEN", NB_TOKEN)
+    is_ascii_or_die("SUBSYSTEM_TOKEN", SUBSYSTEM_TOKEN)
+
+    min_min = max(0, int(args.min_last_detected_minutes))
+    deep = int(args.deep)
+    if deep < 1:
+        deep = 1
+    if deep > MAX_DEEP:
+        logging.warning("[WARN] --deep %s requested, capping to %s", deep, MAX_DEEP)
+        deep = MAX_DEEP
+
+    http = Http(troubleshoot=args.troubleshoot)
+    nb = NetBox(
+        NB_URL,
+        NB_TOKEN,
+        ROLE_CPE,
+        TYPE_FOX100_CPE,
+        TYPE_FOX200,
+        SITE_ID,
+        http,
+        dry_run=args.dry_run,
+    )
+    subsystem = Subsystem(SUBSYSTEM_BASE, SUBSYSTEM_TOKEN, http)
+
+    # --info-only mode (no NetBox changes, no recursion)
+    if args.info_only:
+        rec = subsystem.find_device_by_name(args.device_name.strip())
+        if not rec:
+            print(f"NOT_FOUND {args.device_name.strip()} in Subsystem")
+            return 1
+        print(json.dumps(rec, indent=2, sort_keys=True))
+        return 0
+
+    now_utc = datetime.now(timezone.utc)
+
+    # BFS queue: (name, depth, reason)
+    q: Deque[Tuple[str, int, str]] = deque()
+    seen: Set[str] = set()
+
+    root = args.device_name.strip()
+    q.append((root, 0, "root"))
+    seen.add(root.lower())
+
+    totals = {
+        "processed": 0,
+        "subsystem_found": 0,
+        "netbox_failed_set": 0,
+        "created": 0,
+        "errors": 0,
+        "enqueued": 0,
+    }
+
+    def enqueue(name: str, reason: str, parent_depth: int = 0) -> None:
+        nonlocal q, seen, totals, deep
+        if not name:
+            return
+        k = name.lower()
+        if k in seen:
+            return
+        next_depth = parent_depth + 1
+        if next_depth > deep:
+            return
+        seen.add(k)
+        q.append((name, next_depth, reason))
+        totals["enqueued"] += 1
+        logging.info("  -> enqueue depth=%s name=%s reason=%s", next_depth, name, reason)
+
+    per_device_one_liners: List[str] = []
+
+    try:
+        while q:
+            name, d, reason = q.popleft()
+            totals["processed"] += 1
+            logging.info("[INFO] depth=%s name=%s reason=%s", d, name, reason)
+
+            rec = subsystem.find_device_by_name(name)
+            if rec:
+                totals["subsystem_found"] += 1
+
+                def enq_child(child_name: str, child_reason: str) -> None:
+                    enqueue(child_name, child_reason, parent_depth=d)
+
+                res = sync_from_subsystem_record(
+                    name, rec, nb, subsystem, now_utc, min_min, enq_child
+                )
+                if res.get("netbox_created"):
+                    totals["created"] += 1
+                if (res.get("errors") or 0) > 0:
+                    totals["errors"] += 1
+                per_device_one_liners.append(one_liner_single(res))
+            else:
+                res2 = mark_failed_if_in_netbox_only(name, nb)
+                if res2.get("status_set_failed"):
+                    totals["netbox_failed_set"] += 1
+                if (res2.get("errors") or 0) > 0:
+                    totals["errors"] += 1
+                per_device_one_liners.append(one_liner_single(res2))
+
+    except Exception as e:
+        logging.exception("Unhandled exception in deep sync: %s", e)
+        print(f"{root}: error (exception during deep sync)")
+        return 1
+
+    for line in per_device_one_liners:
+        print(line)
+
+    print(
+        f"SUMMARY: root={root} processed={totals['processed']} "
+        f"subsys_found={totals['subsystem_found']} created={totals['created']} "
+        f"failed_set={totals['netbox_failed_set']} enqueued={totals['enqueued']} errors={totals['errors']}"
+    )
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

files/pppoe_to_dhcp.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+Convert a device config from PPPoE uplink to DHCP on VLAN 4000.
+
+Edits ONLY these paths:
+  .network.zones.wan.mode                         -> "dhcp"
+  .network.zones.wan.alias                        -> []          (ensure present)
+  .network.zones.wan.stp                          -> false       (ensure present)
+  .network.zones.wan.dns                          -> ["41.222.55.1"]
+  .ethernet.ports.eth0.network.vlan_access.enabled-> true
+  .ethernet.ports.eth0.network.vlan_access.id     -> 4000
+
+Everything else is preserved as-is (no key sorting, order preserved).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+from collections import OrderedDict
+from typing import Any, MutableMapping, Sequence
+
+
+def load_json_preserve_order(path: str) -> Any:
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f, object_pairs_hook=OrderedDict)
+
+
+def detect_indent(raw_text: str) -> int:
+    """
+    Best-effort indent detection. Defaults to 2 if unclear.
+    """
+    # Look for the first line that begins with spaces then a quote (a key).
+    m = re.search(r"\n( +)\"", raw_text)
+    if not m:
+        return 2
+    spaces = len(m.group(1))
+    # Common indents are 2 or 4; accept any positive count.
+    return spaces if spaces > 0 else 2
+
+
+def get_mapping(root: Any, path: Sequence[str]) -> MutableMapping[str, Any]:
+    """
+    Walks down dict-like objects; raises KeyError/TypeError if structure is missing.
+    Returns the mapping at the end of the path.
+    """
+    cur = root
+    for key in path:
+        if not isinstance(cur, MutableMapping):
+            raise TypeError(f"Expected object at {'.'.join(path)}, got {type(cur).__name__}")
+        if key not in cur:
+            raise KeyError(f"Missing key '{key}' at {'.'.join(path)}")
+        cur = cur[key]
+    if not isinstance(cur, MutableMapping):
+        raise TypeError(f"Expected object at {'.'.join(path)}, got {type(cur).__name__}")
+    return cur
+
+
+def ensure_path(root: Any, path: Sequence[str]) -> MutableMapping[str, Any]:
+    """
+    Ensures nested dicts exist; creates missing dicts as OrderedDict.
+    Returns the mapping at the end of the path.
+    """
+    cur = root
+    for key in path:
+        if not isinstance(cur, MutableMapping):
+            raise TypeError(f"Expected object while creating {'.'.join(path)}, got {type(cur).__name__}")
+        if key not in cur or cur[key] is None:
+            cur[key] = OrderedDict()
+        cur = cur[key]
+    if not isinstance(cur, MutableMapping):
+        raise TypeError(f"Expected object at {'.'.join(path)}, got {type(cur).__name__}")
+    return cur
+
+
+def set_value(root: Any, path: Sequence[str], value: Any, create: bool = False) -> tuple[bool, Any, Any]:
+    """
+    Set value at path. If create=False, path must exist. If create=True, missing
+    objects along the way are created.
+    Returns (changed, old_value, new_value).
+    """
+    if len(path) < 1:
+        raise ValueError("Path must have at least one key")
+    parent_path = path[:-1]
+    leaf = path[-1]
+
+    parent = ensure_path(root, parent_path) if create else get_mapping(root, parent_path)
+
+    old = parent.get(leaf, None)
+    if old == value:
+        return (False, old, value)
+    parent[leaf] = value
+    return (True, old, value)
+
+
+def validate_post(root: Any) -> None:
+    """
+    Minimal structural and value validation for the changed fields.
+    Raises exceptions on mismatch.
+    """
+    # Check mode
+    wan = get_mapping(root, ["network", "zones", "wan"])
+    if wan.get("mode") != "dhcp":
+        raise ValueError("Post-check failed: .network.zones.wan.mode != 'dhcp'")
+
+    # Check VLAN access
+    vlan_access = get_mapping(root, ["ethernet", "ports", "eth0", "network", "vlan_access"])
+    if vlan_access.get("enabled") is not True:
+        raise ValueError("Post-check failed: vlan_access.enabled is not true")
+    if vlan_access.get("id") != 4000:
+        raise ValueError("Post-check failed: vlan_access.id != 4000")
+
+    # Check schema fields
+    if wan.get("alias") != []:
+        raise ValueError("Post-check failed: .network.zones.wan.alias != []")
+    if wan.get("stp") is not False:
+        raise ValueError("Post-check failed: .network.zones.wan.stp != false")
+
+    # DNS
+    if wan.get("dns") != ["41.222.55.1"]:
+        raise ValueError("Post-check failed: .network.zones.wan.dns != ['41.222.55.1']")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Convert config JSON from PPPoE to DHCP on VLAN 4000 (minimal edits).")
+    ap.add_argument("input", help="Input JSON file (e.g., config.json)")
+    ap.add_argument("-o", "--output", help="Output file. If omitted and --in-place not set, prints to stdout.")
+    ap.add_argument("--in-place", action="store_true", help="Modify input file in place (creates .bak backup).")
+    ap.add_argument("--backup-suffix", default=".bak", help="Backup suffix for --in-place (default: .bak)")
+    args = ap.parse_args()
+
+    in_path = args.input
+
+    raw = open(in_path, "r", encoding="utf-8").read()
+    indent = detect_indent(raw)
+
+    data = load_json_preserve_order(in_path)
+
+    changes: list[str] = []
+
+    def apply(path: Sequence[str], value: Any, create: bool = False) -> None:
+        changed, old, new = set_value(data, path, value, create=create)
+        if changed:
+            changes.append(f"{'.' + '.'.join(path)}: {old!r} -> {new!r}")
+
+    # Required edits
+    apply(["network", "zones", "wan", "mode"], "dhcp", create=False)
+
+    # "Only in new" fields must exist exactly like new
+    apply(["network", "zones", "wan", "alias"], [], create=True)
+    apply(["network", "zones", "wan", "stp"], False, create=True)
+
+    # DNS pinned as requested
+    apply(["network", "zones", "wan", "dns"], ["41.222.55.1"], create=True)
+
+    # VLAN 4000 on eth0
+    apply(["ethernet", "ports", "eth0", "network", "vlan_access", "enabled"], True, create=True)
+    apply(["ethernet", "ports", "eth0", "network", "vlan_access", "id"], 4000, create=True)
+
+    # Validate
+    validate_post(data)
+
+    # Serialize
+    out_text = json.dumps(data, indent=indent, ensure_ascii=False) + "\n"
+
+    # Write
+    if args.in_place:
+        backup_path = in_path + args.backup_suffix
+        if not os.path.exists(backup_path):
+            shutil.copy2(in_path, backup_path)
+        else:
+            # Avoid overwriting an existing backup silently
+            raise FileExistsError(f"Backup already exists: {backup_path}")
+        with open(in_path, "w", encoding="utf-8") as f:
+            f.write(out_text)
+    elif args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(out_text)
+    else:
+        sys.stdout.write(out_text)
+
+    # Report to stderr for CLI usage
+    sys.stderr.write("Applied changes:\n")
+    if changes:
+        for line in changes:
+            sys.stderr.write(f"  - {line}\n")
+    else:
+        sys.stderr.write("  (no changes needed; already in desired state)\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as e:
+        sys.stderr.write(f"ERROR: {e}\n")
+        raise SystemExit(2)
+

files/rabbit-client.sh

@@ -22,6 +22,9 @@ ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
 # Polling interval when no messages
 SLEEP_SECS="${SLEEP_SECS:-1}"
 
+# GO/NO-GO gate bypass (requested): set ignore_gonogo=true to ignore gate
+ignore_gonogo="${ignore_gonogo:-false}"
+
 # Paths
 APP_ROOT="/opt/containers/ansible-worker/app"
 NBPLAY="${APP_ROOT}/bin/nbplay"
@@ -46,6 +49,39 @@ log()  { printf '[consumer] %s\n' "$*"; }
 warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
 err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
 
+# --- GO/NO-GO gate helpers (surgical add) ---
+gate_sleep_secs() {
+  # Random 2..5 seconds inclusive
+  echo $(( (RANDOM % 4) + 2 ))
+}
+
+gate_is_go() {
+  # Bypass if ignore_gonogo is true-ish
+  case "${ignore_gonogo,,}" in
+    true|1|yes|y) return 0 ;;
+  esac
+
+  # Fail-closed: any error/unreachable => NO-GO
+  local body val
+  if ! body="$(curl -fsS --connect-timeout 2 --max-time 3 "http://10.210.12.2:8090/data/go_nogo.txt" 2>/dev/null)"; then
+    return 1
+  fi
+
+  # Must match ^go$ (allow trailing newline in file)
+  val="$(printf '%s' "$body" | tr -d '\r' | head -n1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+  val="${val,,}"
+  [[ "$val" == "go" ]]
+}
+
+gate_block_if_needed() {
+  # If gate is closed, sleep random 2..5 seconds and signal caller to continue loop
+  if gate_is_go; then
+    return 0
+  fi
+  sleep "$(gate_sleep_secs)"
+  return 1
+}
+
 # Safely append string options into an array using eval (so quotes are honored).
 append_opts() {
   local opts_str="$1"
@@ -64,7 +100,6 @@ dispatch_task() {
   task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
   task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
 
-
   # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
   # Supports both historic 'afterupgrade_check' and current 'afterupgrade_indoor_check'
   if [[ "$task" == "afterupgrade_indoor_check" || "$task" == "afterupgrade_check" ]]; then
@@ -78,6 +113,7 @@ dispatch_task() {
     target_ver="$(jq -r '.target_version // ""' <<<"$json")"
     tv_full="$(jq -r '.target_version_full // ""' <<<"$json")"
     schema="$(jq -r '.schema_version // ""' <<<"$json")"
+    is_run_by="$(jq -r '.is_run_by // ""' <<<"$json")"
 
     # Backfill target_version from target_version_full if missing (e.g., fox200-2.2.1-r6801.bin → 2.2.1-r6801)
     if [[ -z "$target_ver" && -n "$tv_full" ]]; then
@@ -96,9 +132,9 @@ dispatch_task() {
     task_options+=" -e target_version='$(esc "$target_ver")'"
     task_options+=" -e target_version_full='$(esc "$tv_full")'"
     task_options+=" -e schema_version='$(esc "$schema")'"
+    task_options+=" -e is_run_by='$(esc "$is_run_by")'"
   fi
 
-
   if [[ -z "$device" || -z "$task" ]]; then
     warn "payload missing required keys (inscope_device/task_name). Skipping."
     return 0
@@ -109,8 +145,15 @@ dispatch_task() {
   local -a extra_nbplay_opts=()
   local n=""
 
+  # NEW: allow trailing "_force" suffix (can be combined with other suffixes like _tonight / _<hours>)
+  if [[ "$base_task" =~ ^(.+)_force$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+    extra_nbplay_opts+=("-eforce_upgrade=yes")
+    log "Parsed _force suffix: base='${base_task}' (passed as -e force_upgrade=yes)"
+  fi
+
   # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4
-  if [[ "$task" =~ ^(.+)_tonight$ ]]; then
+  if [[ "$base_task" =~ ^(.+)_tonight$ ]]; then
     base_task="${BASH_REMATCH[1]}"
 
     # now, today 01:00, tomorrow 01:00 (local time)
@@ -129,26 +172,45 @@ dispatch_task() {
     rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
     total_h=$(( ceil_h + rnd - 1 ))
 
+    # NEW: if rebootin is huge (>=18h), convert to immediate reboot (0h)
+    if (( total_h >= 19 )); then
+      total_h=0
+      log "Adjusted _tonight rebootin to 0h because computed value was >=18h"
+    fi
+
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
-  elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+  elif [[ "$base_task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
     n="${BASH_REMATCH[1]}"
     base_task="update-rebootin"
     extra_nbplay_opts+=("-erebootin=${n}")
-  elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
+  elif [[ "$base_task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
     n="${BASH_REMATCH[1]}"
     base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
     extra_nbplay_opts+=("-erebootin=${n}")
-  elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
+  elif [[ "$base_task" =~ ^update-reboot-scheduler_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-reboot-scheduler"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$base_task" =~ ^(update-reboot[0-9]{3}-scheduler)_([0-9]{1,4})$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+    n="${BASH_REMATCH[2]}"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$base_task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
     # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin=<n>
     n="${BASH_REMATCH[1]}"
     base_task="update-indoor"
     extra_nbplay_opts+=("-erebootin=${n}")
     log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})"
+  elif [[ "$base_task" =~ ^update-indoor-bootenv_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor-bootenv"
+    extra_nbplay_opts+=("-erebootin=${n}")
+    log "Parsed update-indoor-bootenv suffix: ${n}h (passed as -e rebootin=${n})"
   fi
 
   # --- Default rebootin for reboot-family when not explicitly provided ---
-  if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot)$ ]]; then
+  if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot|update-reboot-scheduler|update-reboot[0-9]{3}-scheduler)$ ]]; then
     # only set if neither task_options nor extra_nbplay_opts already contain rebootin
     if [[ "$task_options" != *"rebootin="* ]] && ! printf '%s\n' "${extra_nbplay_opts[@]}" | grep -q 'rebootin='; then
       # restore legacy behavior: immediate reboot if none specified
@@ -209,6 +271,11 @@ if [[ -n "$EXCHANGE" ]]; then
   # Single-queue consume loop (unchanged branch)
   log "Press Ctrl+C to stop."
   while :; do
+    # GO/NO-GO gate: do NOT dequeue unless gate is GO
+    if ! gate_block_if_needed; then
+      continue
+    fi
+
     RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
       "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
     }')"
@@ -258,6 +325,11 @@ else
 
   # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
   while :; do
+    # GO/NO-GO gate: do NOT dequeue unless gate is GO
+    if ! gate_block_if_needed; then
+      continue
+    fi
+
     local_got_message=0
 
     for Q in "${QUEUE_LIST[@]}"; do
@@ -301,4 +373,4 @@ else
       sleep "$SLEEP_SECS"
     fi
   done
-fi
+fi

files/startblock.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+set -eu
+
+/bin/sh /root/stopblock.sh >/dev/null 2>&1 || true
+sleep 1
+#grep -q "block" /etc/rc.local || cat /root/rc.local-with > /etc/rc.local
+grep -qi "block" /etc/rc.local || sed -i '/^exit 0$/i\/root/startblock.sh' /etc/rc.local
+
+sleep 1
+echo 10 > /proc/sys/net/netfilter/nf_conntrack_max ; sleep 4 ; echo 7000 > /proc/sys/net/netfilter/nf_conntrack_max
+
+LAN_IF="br-lan"
+LAN_NET="192.168.2.0/24"
+EXCLUDE_IP="13.244.149.112/32"
+
+PORTAL_IP="102.38.126.180"
+PORTAL_HTTP_PORT="8082"
+PORTAL_HTTPS_PORT="8083"
+
+# -------------------------
+# NAT CAPTURE
+# -------------------------
+
+iptables -t nat -N CAPTIVE 2>/dev/null || true
+iptables -t nat -F CAPTIVE
+
+while iptables -t nat -C PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE 2>/dev/null; do
+  iptables -t nat -D PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE
+done
+iptables -t nat -I PREROUTING 1 -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE
+
+iptables -t nat -A CAPTIVE -d "$EXCLUDE_IP" -j RETURN
+
+iptables -t nat -A CAPTIVE -p udp --dport 53 -j REDIRECT --to-ports 53
+iptables -t nat -A CAPTIVE -p tcp --dport 53 -j REDIRECT --to-ports 53
+
+iptables -t nat -A CAPTIVE -p tcp --dport 80  -j DNAT --to-destination ${PORTAL_IP}:${PORTAL_HTTP_PORT}
+iptables -t nat -A CAPTIVE -p tcp --dport 443 -j DNAT --to-destination ${PORTAL_IP}:${PORTAL_HTTPS_PORT}
+
+# -------------------------
+# FILTER ENFORCEMENT (BLOCK EVERYTHING ELSE)
+# -------------------------
+
+iptables -N CAPTIVE_BLOCK 2>/dev/null || true
+iptables -F CAPTIVE_BLOCK
+
+# Allow established traffic
+iptables -A CAPTIVE_BLOCK -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+
+# Allow DNS to router itself (client -> router)
+iptables -A CAPTIVE_BLOCK -p udp -d 192.168.2.1 --dport 53 -j ACCEPT
+iptables -A CAPTIVE_BLOCK -p tcp -d 192.168.2.1 --dport 53 -j ACCEPT
+
+# Allow access to captive portal server (after DNAT)
+iptables -A CAPTIVE_BLOCK -p tcp -d "$PORTAL_IP" --dport "$PORTAL_HTTP_PORT" -j ACCEPT
+iptables -A CAPTIVE_BLOCK -p tcp -d "$PORTAL_IP" --dport "$PORTAL_HTTPS_PORT" -j ACCEPT
+
+# Allow bypass destination fully (optional but usually desired)
+iptables -A CAPTIVE_BLOCK -d "$EXCLUDE_IP" -j ACCEPT
+
+# Block everything else from LAN clients
+iptables -A CAPTIVE_BLOCK -j REJECT --reject-with icmp-admin-prohibited
+
+# Hook CAPTIVE_BLOCK into FORWARD (ensure single jump)
+while iptables -C FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK 2>/dev/null; do
+  iptables -D FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK
+done
+iptables -I FORWARD 1 -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK
+
+echo "OK: filter rules applied"
+

files/stopblock.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -eu
+#cat /root/rc.local-with | grep -v block > /etc/rc.local
+
+grep -qi "block" /etc/rc.local && sed -i '\|^/root/startblock\.sh$|d' /etc/rc.local
+
+LAN_IF="br-lan"
+LAN_NET="192.168.2.0/24"
+
+# -------------------------
+# FILTER: remove CAPTIVE_BLOCK hook + chain
+# -------------------------
+
+# Remove any FORWARD jumps that match our hook exactly
+while iptables -C FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK 2>/dev/null; do
+  iptables -D FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK
+done
+
+# Flush/delete CAPTIVE_BLOCK chain if present
+iptables -F CAPTIVE_BLOCK 2>/dev/null || true
+iptables -X CAPTIVE_BLOCK 2>/dev/null || true
+
+# -------------------------
+# NAT: remove CAPTIVE hook + chain
+# -------------------------
+
+# Remove any PREROUTING jumps that match our hook exactly
+while iptables -t nat -C PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE 2>/dev/null; do
+  iptables -t nat -D PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE
+done
+
+# Flush/delete CAPTIVE chain if present
+iptables -t nat -F CAPTIVE 2>/dev/null || true
+iptables -t nat -X CAPTIVE 2>/dev/null || true
+
+echo "OK: filter rules removed"