1129

files/ansible-playbooks/afterupgrade_check.yml

@@ -24,8 +24,7 @@
     max_attempts_default: 3
 
     # --- Hardcoded cloud API bearer (per request) ---
-    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
-
   tasks:
     # ---- Normalize metadata safely (no self-referential defaults) ----
     - name: Normalize metadata (no clever transforms)
@@ -37,6 +36,14 @@
         target_version: "{{ target_version | default('') }}"
         # preserve the original string verbatim for all subsequent retries
         target_version_full: "{{ target_version | default('') }}"
+        is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+
+    - name: Debug is_run_by mode
+      ansible.builtin.debug:
+        msg:
+          - "is_run_by={{ is_run_by | default('UNSET') }}"
+          - "is_run_by_effective={{ is_run_by_effective }}"
 
     - name: Derive effective target version (avoid extra-var masking)
       ansible.builtin.set_fact:
@@ -145,7 +152,7 @@
       ignore_errors: true
 
     - name: Set eth0_macaddress fact
-      when: mac_probe is defined and mac_probe.rc == 0
+      when: mac_probe is defined and (mac_probe.rc | default(1)) == 0
       ansible.builtin.set_fact:
         eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
 
@@ -166,7 +173,7 @@
       ansible.builtin.shell: |
         set -e
         curl -sS -L --request PATCH --post301 --post302 \
-          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          "https://cloud.ikeja.co.za/v2/external/devices/{{ enc_mac }}/bandwidth-control" \
           --header "Authorization: Bearer {{ cloud_api_bearer }}" \
           --header "Content-Type: application/json" \
           --header "Accept: application/json" \
@@ -223,6 +230,118 @@
       register: rmq_j_success
       changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
 
+    - name: Success | Publish action_state done
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'done' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success | Clear action_restart_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_restart_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success | Clear action_next
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+      
+    - name: Success | Clear action_next_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success | Pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+      
     # NEW: send a control tag to clean up device state on success
     - name: Build cleanup control payload (update_cleanup_success)
       when: journal_success_payload is defined
@@ -232,6 +351,7 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "update_cleanup_success"
 
+
     - name: Publish cleanup control message to control queue
       when: control_cleanup_payload is defined
       delegate_to: localhost
@@ -254,6 +374,50 @@
       register: rmq_cleanup_success
       changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
 
+    # Scheduled success only: chain next step (set action_next + trigger worker)
+    - name: Scheduled success | Set action_next to sot-updater-scheduler
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success | Publish sot-updater-scheduler work message
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
     # Mismatch: reachable & banner read, but not equal to target_version
     - name: Build mismatch journal payload
       when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
@@ -289,6 +453,34 @@
       register: rmq_j_mismatch
       changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
 
+    - name: Failure | Pause before action_state failed
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_mismatch_payload is defined
+
+    - name: Failure | Publish action_state failed (mismatch)
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'failed' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     # SSH error path: TCP OK, but SSH failed
     - name: Build failure journal payload (ssh error) + mark retry
       when: nc_probe.rc == 0 and banner_probe.rc != 0
@@ -375,6 +567,34 @@
       register: rmq_j_gaveup
       changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
 
+    - name: Failure | Pause before action_state failed (gave up)
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_gaveup_payload is defined
+
+    - name: Failure | Publish action_state failed (gave up)
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'failed' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     - name: Stop host after final gave-up
       when: journal_gaveup_payload is defined
       ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_indoor_check.yml

@@ -344,7 +344,8 @@
       delegate_to: localhost
       ansible.builtin.shell: |
         sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
+          -o PreferredAuthentications=password -o PasswordAuthentication=yes -o NumberOfPasswordPrompts=1 \
+          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o PubkeyAuthentication=no -o ConnectTimeout=10 \
           root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
       register: dev2_fwver
       changed_when: false
@@ -370,6 +371,14 @@
         effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
         correlation_id: "{{ correlation_id | default('') }}"
         original_emitted_at: "{{ original_emitted_at | default('') }}"
+        is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+
+    - name: Debug is_run_by mode
+      ansible.builtin.debug:
+        msg:
+          - "is_run_by={{ is_run_by | default('UNSET') }}"
+          - "is_run_by_effective={{ is_run_by_effective }}"
 
     - name: Compute read_ok flag based on firmware readout
       delegate_to: localhost
@@ -584,6 +593,13 @@
       register: rmq_tag_remove_sched
       changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
 
+    - name: Success pause before NetBox wrapup custom fields 1
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
     # --- Normalize firmware string and set custom field on success ---
     - name: Capture raw firmware banner for normalization on success path
       when: journal_success_payload is defined
@@ -641,6 +657,197 @@
       register: rmq_customfield_fw
       changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
 
+    - name: Success publish action_state done
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'done' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success clear action_restart_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_restart_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success clear action_next
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Success clear action_next_timestamp
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Build cleanup control payload update_cleanup_success
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    - name: Success pause before NetBox wrapup custom fields
+      ansible.builtin.pause:
+        seconds: 3
+      delegate_to: localhost
+      changed_when: false
+      when: journal_success_payload is defined
+
+    - name: Scheduled success set action_next to sot-updater-scheduler
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success publish sot-updater-scheduler work message
+      when: journal_success_payload is defined and (is_run_by_effective | default('manual')) == 'scheduler'
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     # MISMATCH path
     - name: Build journal payload for version mismatch after upgrade
       when: (read_ok | bool) and (not (version_match | bool))
@@ -676,6 +883,56 @@
       register: rmq_j_mismatch
       changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
 
+    - name: Failure pause before action_state failed mismatch
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_mismatch_payload is defined
+
+    - name: Failure publish action_state failed mismatch
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'ready' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    - name: Failure publish action_next sot-updater-scheduler mismatch
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     - name: Stop host after mismatch path is handled
       when: journal_mismatch_payload is defined
       ansible.builtin.meta: end_host
@@ -754,6 +1011,56 @@
       register: rmq_pub_gaveup
       changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
 
+    - name: Failure pause before action_state failed gaveup
+      ansible.builtin.pause:
+        seconds: 1
+      delegate_to: localhost
+      changed_when: false
+      when: journal_gaveup_payload is defined
+
+    - name: Failure publish action_state failed gaveup
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': 'ready' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+    - name: Failure publish action_next sot-updater-scheduler gaveup
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
     # Only schedule next attempt if budget left
     - name: Build delayed payload for next indoor attempt wait ten minutes
       when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))

files/ansible-playbooks/afterupgrade_withcurl.yml

@@ -37,7 +37,7 @@
     max_attempts_default: 3
 
     # --- Hardcoded cloud API bearer (per request) ---
-    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
 
   tasks:
     # ---- Normalize metadata safely (no self-referential defaults) ----
@@ -179,7 +179,7 @@
       ansible.builtin.shell: |
         set -e
         curl -sS -L --request PATCH --post301 --post302 \
-          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          "https://cloud.ikeja.co.za/v2/external/devices/{{ enc_mac }}/bandwidth-control" \
           --header "Authorization: Bearer {{ cloud_api_bearer }}" \
           --header "Content-Type: application/json" \
           --header "Accept: application/json" \

files/ansible-playbooks/connstats-deploy.yml

@@ -0,0 +1,202 @@
+---
+- name: Deploy connstats (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    # Cron line we must ensure (preserve any other lines)
+    connstats_cron_line: "*/10 * * * * /root/connstats.sh --debug --always-find-offset  >/dev/null 2>&1"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+            ############ step 2
+
+            - name: Compute MD5 of local connstats.sh
+              delegate_to: localhost
+              command: md5sum files/connstats.sh
+              register: md5_local_cstats
+              changed_when: false
+
+            - name: Compute MD5 of remote /root/connstats.sh
+              raw: "md5sum /root/connstats.sh || busybox md5sum /root/connstats.sh"
+              register: md5_remote_cstats
+              changed_when: false
+              failed_when: false
+
+            - name: Decide if connstats.sh needs upload
+              set_fact:
+                upload_cstats: >-
+                  {{ (md5_remote_cstats.rc != 0)
+                     or ((md5_local_cstats.stdout.split()[0])
+                         != (md5_remote_cstats.stdout.split()[0] if (md5_remote_cstats.stdout is defined) else '')) }}
+
+            - name: Upload connstats.sh via scp (overwrite if changed)
+              when: upload_cstats | bool
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/connstats.sh
+                {{ ssh_user }}@{{ ansible_host }}:/root/connstats.sh
+              register: scp_connstats
+              retries: 3
+              delay: 2
+              until: scp_connstats.rc == 0
+
+            - name: Ensure /root/connstats.sh is executable and owned by root
+              raw: |
+                chown root:root /root/connstats.sh && chmod 0755 /root/connstats.sh
+
+            ############ step 3
+
+            - name: Ensure /etc/crontabs/root exists (touch with perms)
+              raw: |
+                if [ ! -f /etc/crontabs/root ]; then
+                    touch /etc/crontabs/root
+                fi
+                chown root:root /etc/crontabs/root
+                chmod 0644 /etc/crontabs/root
+
+            - name: Check if connstats cron line already present
+              raw: |
+                grep -Eq '^\*/10[[:space:]]+\*[[:space:]]+\*[[:space:]]+\*[[:space:]]+\*[[:space:]]+/root/connstats\.sh[[:space:]]+--debug[[:space:]]+--always-find-offset([[:space:]]+>/dev/null[[:space:]]+2>&1)?[[:space:]]*$' /etc/crontabs/root
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Upload snippet connstats-crond-root to /tmp (only if missing)
+              when: cron_grep.rc != 0
+              delegate_to: localhost
+              command: >
+                sshpass -p {{ ssh_pass | quote }}
+                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                files/connstats-crond-root
+                {{ ssh_user }}@{{ ansible_host }}:/tmp/connstats-crond-root.snippet
+
+            - name: Append snippet to /etc/crontabs/root (only if missing)
+              when: cron_grep.rc != 0
+              raw: |
+                cat /tmp/connstats-crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/connstats-crond-root.snippet
+              register: cron_append
+              changed_when: true
+
+            - name: Set result status (success deployed or no change)
+              set_fact:
+                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_cstats | bool) else 'SUCCESS_NO_CHANGE' }}"
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # custom field update (per your sample)
+        - name: Publish custom-field update connstats deployed to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': inscope_device_name,
+                'task_name': 'custom_field_set',
+                'task_add1': 'connstats',
+                'task_result': 'deployed'
+              } | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: false
+
+        # final wrap-up journal "connstats: ..." with actions performed
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + ((upload_cstats | default(false) | bool)                         | ternary(['uploaded connstats.sh'], []))
+                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added connstats crontab entry'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                connstats: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/connstats-remove.yml

@@ -0,0 +1,179 @@
+---
+- name: Remove connstats (single device, linear)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
+
+    # RabbitMQ (use controls exchange + queue_controls like the reference)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+
+    - block:
+
+        # --- SSH reachability check ---
+        - name: Check SSH connectivity (raw ping)
+          raw: "echo ping"
+          register: ping_result
+          ignore_errors: true
+
+        - block:
+
+            ############ step 2
+
+            - name: Check if connstats cron line is present
+              raw: |
+                if [ -f /etc/crontabs/root ]; then
+                    grep -Eq '/root/connstats\.sh([[:space:]]|$)' /etc/crontabs/root
+                else
+                    exit 1
+                fi
+              register: cron_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Remove connstats cron line (preserve any other lines)
+              when: cron_grep.rc == 0
+              raw: |
+                sed -i '\|/root/connstats\.sh|d' /etc/crontabs/root
+              register: cron_remove
+              changed_when: true
+
+            - name: Check if /root/connstats.log exists
+              raw: "test -e /root/connstats.log"
+              register: connstats_log_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Remove /root/connstats.log
+              when: connstats_log_grep.rc == 0
+              raw: "rm -f /root/connstats.log"
+              register: connstats_log_remove
+              changed_when: true
+
+            - name: Check if /root/connstats.sh exists
+              raw: "test -e /root/connstats.sh"
+              register: connstats_script_grep
+              failed_when: false
+              changed_when: false
+
+            - name: Remove /root/connstats.sh
+              when: connstats_script_grep.rc == 0
+              raw: "rm -f /root/connstats.sh"
+              register: connstats_script_remove
+              changed_when: true
+
+            - name: Set result status (success removed or no change)
+              set_fact:
+                result_status: >-
+                  {{ 'SUCCESS_REMOVED'
+                     if ((cron_grep.rc == 0)
+                         or (connstats_log_grep.rc == 0)
+                         or (connstats_script_grep.rc == 0))
+                     else 'SUCCESS_NO_CHANGE' }}
+
+          when: ping_result is succeeded
+
+        - name: Set status fact (no ssh)
+          when: ping_result is failed
+          set_fact:
+            result_status: "NO_SSH"
+
+      rescue:
+        - name: Mark result as failed
+          set_fact:
+            result_status: "FAILED during {{ ansible_failed_task.name }}"
+
+      always:
+
+        - name: Compute inscope device
+          set_fact:
+            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+
+        # custom field update (per your sample)
+        - name: Publish custom-field update connstats removed to control queue
+          when: result_status == 'SUCCESS_REMOVED' or result_status == 'SUCCESS_NO_CHANGE'
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers: { content-type: "application/json" }
+            body_format: json
+            body:
+              properties: { content_type: "application/json" }
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': inscope_device_name,
+                'task_name': 'custom_field_set',
+                'task_add1': 'connstats',
+                'task_result': 'removed'
+              } | to_json }}"
+              payload_encoding: "string"
+          register: rmq_cf
+          changed_when: false
+
+        # final wrap-up journal "connstats: ..." with actions performed
+        - name: Build actions list
+          set_fact:
+            _actions_list: >-
+              {{
+                []
+                + (((cron_grep is defined) and ((cron_grep.rc | default(1)) == 0))                  | ternary(['removed connstats crontab entry'], []))
+                + (((connstats_log_grep is defined) and ((connstats_log_grep.rc | default(1)) == 0)) | ternary(['removed connstats.log'], []))
+                + (((connstats_script_grep is defined) and ((connstats_script_grep.rc | default(1)) == 0)) | ternary(['removed connstats.sh'], []))
+              }}
+
+        - name: Build actions string
+          set_fact:
+            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
+
+        - name: Build wrap-up journal payload
+          delegate_to: localhost
+          set_fact:
+            wrap_payload:
+              inscope_device: "{{ inscope_device_name }}"
+              task_name: "journal_add"
+              task_result: >-
+                connstats: {{ 'success' if (result_status == 'SUCCESS_REMOVED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
+                — actions: {{ _actions_str }}
+
+        - name: Publish wrap-up journal to control queue
+          delegate_to: localhost
+          uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ wrap_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_wrap
+          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
+
+        # Local summary (kept for operator visibility)
+        - name: Summary
+          debug:
+            msg:
+              - "result_status: {{ result_status }}"
+              - "we're good"

files/ansible-playbooks/connstats-scheduler-remover.yml

@@ -0,0 +1,244 @@
+# connstats-scheduler-remover.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run connstats-remove.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: connstats-remove.yml
+  vars:
+    is_run_by: "scheduler"
+
+
+- name: Phase 4 | NetBox wrapup and chaining for connstats removal
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: Wrapper compute connstats removal outcome
+      set_fact:
+        inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+        connstats_ok: "{{ (result_status | default('') | regex_search('^SUCCESS_')) is not none }}"
+        connstats_status: "{{ result_status | default('UNKNOWN') }}"
+      changed_when: false
+
+    - name: Wrapper pause before clear action_next_timestamp
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper clear action_next_timestamp
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before set action_state
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_state
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': (connstats_ok | ternary('done','failed')) } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before chaining to sot updater
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_next to sot-updater-scheduler
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper publish sot-updater-scheduler work message
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before final journal
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper final journal report
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'journal_add', 'task_result': (connstats_ok | ternary('connstats: successfully removed','connstats: removal failed')) ~ ' (' ~ connstats_status ~ ')' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/connstats-scheduler.yml

@@ -0,0 +1,244 @@
+# connstats-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run connstats-deploy.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: connstats-deploy.yml
+  vars:
+    is_run_by: "scheduler"
+
+
+- name: Phase 4 | NetBox wrapup and chaining for connstats
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: Wrapper compute connstats outcome
+      set_fact:
+        inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+        connstats_ok: "{{ (result_status | default('') | regex_search('^SUCCESS_')) is not none }}"
+        connstats_status: "{{ result_status | default('UNKNOWN') }}"
+      changed_when: false
+
+    - name: Wrapper pause before clear action_next_timestamp
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper clear action_next_timestamp
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before set action_state
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_state
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': (connstats_ok | ternary('done','failed')) } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before chaining to sot updater
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_next to sot-updater-scheduler
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper publish sot-updater-scheduler work message
+      when: connstats_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before final journal
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper final journal report
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'journal_add', 'task_result': (connstats_ok | ternary('connstats: successfully deployed','connstats: deployment failed')) ~ ' (' ~ connstats_status ~ ')' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/ptsd-migrate-wrapper.yml

@@ -0,0 +1,72 @@
+# ptsd-migrate-wrapper.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run full ptsd-migrate.yml unchanged
+
+- hosts: all
+  gather_facts: no
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('(?:^|\\s)ip=([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)(?:\\s|$)', '\\1') | default('') }}"
+        _ip: >-
+          {%- if _ip_raw is string -%}
+            {{ _ip_raw }}
+          {%- elif _ip_raw is sequence and (_ip_raw | length) > 0 -%}
+            {{ _ip_raw[0] }}
+          {%- else -%}
+            ""
+          {%- endif -%}
+      set_fact:
+        wrapper_ok_line: "{{ _ok_line }}"
+        wrapper_fail_line: "{{ _fail_line }}"
+        wrapper_nb_ip: "{{ _ip | trim }}"
+
+    - name: "Wrapper | Debug extracted values"
+      ansible.builtin.debug:
+        msg:
+          - "wrapper_ok_line='{{ wrapper_ok_line }}'"
+          - "wrapper_fail_line='{{ wrapper_fail_line | default('') }}'"
+          - "wrapper_nb_ip='{{ wrapper_nb_ip }}'"
+
+    - name: "Wrapper | Stop host if cloud reports no fresh IP"
+      when: (wrapper_fail_line | default('') | length) > 0
+      meta: end_host
+
+    - name: "Wrapper | Assert IP extracted successfully"
+      ansible.builtin.assert:
+        that:
+          - nb_script is defined
+          - nb_script.stat.exists | default(false)
+          - (wrapper_ok_line | length) > 0
+          - (wrapper_nb_ip | length) > 0
+        fail_msg: >-
+          Wrapper stop: could not extract IP from nb_onedevice_update.py output.
+          ok_line='{{ wrapper_ok_line | default('') }}'
+          fail_line='{{ wrapper_fail_line | default('') }}'
+          stdout_tail={{ (nb_preflight.stdout_lines | default([]))[-12:] }}
+
+    - name: "Wrapper | Override ansible_host to refreshed IP ({{ wrapper_nb_ip }})"
+      set_fact:
+        ansible_host: "{{ wrapper_nb_ip }}"
+
+    - name: "Wrapper | Reset SSH connection context (ensure new ansible_host is used)"
+      meta: reset_connection
+
+
+- import_playbook: ptsd-migrate.yml

files/ansible-playbooks/ptsd-migrate.yml

@@ -79,7 +79,7 @@
         {{ pathprefix }}
         DIGITS="{{ dev2_lldp_digits }}";
         cat /var/run/lldp_server.json 2>/dev/null \
-          | grep "${DIGITS}" -A 10 \
+          | grep -Ei "ikeja${DIGITS}\"" -A 10 \
           | grep address \
           | grep -vE 'subtype|ipv6' \
           | awk -F'"' '{ print $4 }' \
@@ -93,7 +93,7 @@
         {{ pathprefix }}
         DIGITS="{{ dev2_lldp_digits }}";
         cat /var/run/lldp_server.json 2>/dev/null \
-          | grep "${DIGITS}" -A 15 \
+          | grep -Ei "ikeja${DIGITS}\"" -A 15 \
           | grep 'address_ipv6' \
           | awk -F'"' '{ print $4 }' \
           | head -n1
@@ -1175,7 +1175,7 @@
         ptsd_migrate_report: "{{ (ptsd_migrate_report | default('')) ~ (', ' if (ptsd_migrate_report | default('')) != '' else '') ~ ('cleanup done') }}"
       changed_when: false
 
-- name: PHASE X | Wait 45 seconds after config activation
+- name: PHASE X | Wait 30 seconds after config activation
   hosts: all
   gather_facts: false
   vars:
@@ -1188,12 +1188,12 @@
     control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
   tasks:
     - ansible.builtin.pause:
-        seconds: 45
+        seconds: 30
 
     - name: Report record post activation wait
       delegate_to: localhost
       ansible.builtin.set_fact:
-        ptsd_migrate_report: "{{ (ptsd_migrate_report | default('')) ~ (', ' if (ptsd_migrate_report | default('')) != '' else '') ~ ('wait 45s') }}"
+        ptsd_migrate_report: "{{ (ptsd_migrate_report | default('')) ~ (', ' if (ptsd_migrate_report | default('')) != '' else '') ~ ('wait 30s') }}"
       changed_when: false
 
     - name: Final migrate cumulative report

files/ansible-playbooks/ptsd_reacquire_attempt.yml

@@ -90,7 +90,7 @@
         {{ pathprefix }}
         DIGITS="{{ dev2_lldp_digits }}";
         cat /var/run/lldp_server.json 2>/dev/null \
-          | grep "${DIGITS}" -A 10 \
+          | grep -Ei "ikeja${DIGITS}" -A 10 \
           | grep address \
           | grep -vE 'subtype|ipv6' \
           | awk -F'"' '{ print $4 }' \
@@ -99,12 +99,18 @@
       changed_when: false
       failed_when: false
 
+    - name: Debug raw DEV2 LLDP IPv6 command output
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        var: dev2_lldp_ip6_raw.stdout_lines      
+
     - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
       ansible.builtin.raw: >
         {{ pathprefix }}
         DIGITS="{{ dev2_lldp_digits }}";
         cat /var/run/lldp_server.json 2>/dev/null \
-          | grep "${DIGITS}" -A 15 \
+          | grep -Ei "ikeja${DIGITS}" -A 15 \
           | grep 'address_ipv6' \
           | awk -F'"' '{ print $4 }' \
           | head -n1
@@ -118,6 +124,14 @@
         lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
       changed_when: false
 
+    - name: Debug captured DEV2 LLDP IPv6 fact
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_lldp_ip6_raw={{ dev2_lldp_ip6_raw.stdout | default('') | trim }}"
+          - "lldp_dev2_ip6={{ lldp_dev2_ip6 | default('') | trim }}"
+
     - name: Classify LLDP IPv4 candidate
       delegate_to: localhost
       ansible.builtin.set_fact:

files/ansible-playbooks/restart-cloud-agent-long-wrapper.yml

@@ -0,0 +1,44 @@
+# restart-cloud-agent-wrapper.yml
+# Phase 1: Subsystem -> NetBox sync for current inventory_hostname
+# Phase 2: Restart cloud-agent on the device
+
+- name: Phase 1 | Subsystem -> NetBox sync before cloud-agent restart
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: "Wrapper | Verify netbox_subsystem_ikejanum_recursive.py exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: ./netbox_subsystem_ikejanum_recursive.py
+      register: subsystem_script
+      changed_when: false
+
+    - name: "Wrapper | Run netbox_subsystem_ikejanum_recursive.py for {{ inventory_hostname }}"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -o pipefail
+        python3 -u ./netbox_subsystem_ikejanum_recursive.py "{{ inventory_hostname }}" 2>&1
+      args:
+        executable: /bin/bash
+      register: subsystem_sync
+      changed_when: false
+      failed_when: false
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Show subsystem sync result"
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "script_exists={{ subsystem_script.stat.exists | default(false) }}"
+          - "rc={{ subsystem_sync.rc | default('NA') }}"
+          - "stdout_lines={{ subsystem_sync.stdout_lines | default([]) }}"
+          - "stderr_lines={{ subsystem_sync.stderr_lines | default([]) }}"
+          - "raw stdout (joined): {{ subsystem_sync.stdout | default('') }}"
+          - "raw stderr (joined): {{ subsystem_sync.stderr | default('') }}"
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Abort softly if script missing"
+      ansible.builtin.meta: end_host
+      when: not (subsystem_script.stat.exists | default(false))
+
+- import_playbook: restart-cloud-agent-long.yml

files/ansible-playbooks/restart-cloud-agent-long.yml

@@ -0,0 +1,25 @@
+---
+- name: Temporarily move cloud-agent and restore it
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Move /tmp/launchd/services/cloud-agent to /root/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /tmp/launchd/services/cloud-agent /root/
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 200
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /root/cloud-agent /tmp/launchd/services/
+      register: move_back
+      changed_when: true

files/ansible-playbooks/restart-cloud-agent-wrapper.yml

@@ -0,0 +1,44 @@
+# restart-cloud-agent-wrapper.yml
+# Phase 1: Subsystem -> NetBox sync for current inventory_hostname
+# Phase 2: Restart cloud-agent on the device
+
+- name: Phase 1 | Subsystem -> NetBox sync before cloud-agent restart
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: "Wrapper | Verify netbox_subsystem_ikejanum_recursive.py exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: ./netbox_subsystem_ikejanum_recursive.py
+      register: subsystem_script
+      changed_when: false
+
+    - name: "Wrapper | Run netbox_subsystem_ikejanum_recursive.py for {{ inventory_hostname }}"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -o pipefail
+        python3 -u ./netbox_subsystem_ikejanum_recursive.py "{{ inventory_hostname }}" 2>&1
+      args:
+        executable: /bin/bash
+      register: subsystem_sync
+      changed_when: false
+      failed_when: false
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Show subsystem sync result"
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "script_exists={{ subsystem_script.stat.exists | default(false) }}"
+          - "rc={{ subsystem_sync.rc | default('NA') }}"
+          - "stdout_lines={{ subsystem_sync.stdout_lines | default([]) }}"
+          - "stderr_lines={{ subsystem_sync.stderr_lines | default([]) }}"
+          - "raw stdout (joined): {{ subsystem_sync.stdout | default('') }}"
+          - "raw stderr (joined): {{ subsystem_sync.stderr | default('') }}"
+      when: subsystem_script.stat.exists | default(false)
+
+    - name: "Wrapper | Abort softly if script missing"
+      ansible.builtin.meta: end_host
+      when: not (subsystem_script.stat.exists | default(false))
+
+- import_playbook: restart-cloud-agent.yml

files/ansible-playbooks/sot-updater-scheduler.yml

@@ -0,0 +1,194 @@
+# sot-updater-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run full sot-updater-current.yml
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('(?:^|\\s)ip=([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)(?:\\s|$)', '\\1') | default('') }}"
+        _ip: >-
+          {%- if _ip_raw is string -%}
+            {{ _ip_raw }}
+          {%- elif _ip_raw is sequence and (_ip_raw | length) > 0 -%}
+            {{ _ip_raw[0] }}
+          {%- else -%}
+            ""
+          {%- endif -%}
+      set_fact:
+        wrapper_ok_line: "{{ _ok_line }}"
+        wrapper_fail_line: "{{ _fail_line }}"
+        wrapper_nb_ip: "{{ _ip | trim }}"
+
+
+    - name: "Wrapper | Debug extracted values"
+      ansible.builtin.debug:
+        msg:
+          - "wrapper_ok_line='{{ wrapper_ok_line }}'"
+          - "wrapper_fail_line='{{ wrapper_fail_line | default('') }}'"
+          - "wrapper_nb_ip='{{ wrapper_nb_ip }}'"
+
+    # ----------------------- Option A: Cloud offline / no IP -> Journal + stop cleanly -----------------------
+    - name: "Wrapper | Journal: device not online in cloud (skip full scan)"
+      delegate_to: localhost
+      when: (wrapper_fail_line | default('') | length) > 0
+      vars:
+        _journal_msg: >-
+          during sot-updater scan the device was not online in cloud (no IP). please try later.
+          details: {{ wrapper_fail_line | default('') }}
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': _journal_msg
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: "Wrapper | Mark host to skip full scan (cloud-offline)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      set_fact:
+        wrapper_skip_full_scan: true
+
+
+    - name: "Wrapper | Stop host after cloud-offline journal (no full scan)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      meta: end_host
+
+    # ----------------------- Normal path: we have OK ip=... -> proceed -----------------------
+    - name: "Wrapper | Assert IP extracted successfully"
+      ansible.builtin.assert:
+        that:
+          - nb_script is defined
+          - nb_script.stat.exists | default(false)
+          - (wrapper_ok_line | length) > 0
+          - (wrapper_nb_ip | length) > 0
+        fail_msg: >-
+          Wrapper stop: could not extract IP from nb_onedevice_update.py output.
+          ok_line='{{ wrapper_ok_line | default('') }}'
+          fail_line='{{ wrapper_fail_line | default('') }}'
+          stdout_tail={{ (nb_preflight.stdout_lines | default([]))[-12:] }}
+
+    - name: "Wrapper | Override ansible_host to refreshed IP ({{ wrapper_nb_ip }})"
+      set_fact:
+        ansible_host: "{{ wrapper_nb_ip }}"
+
+    - name: "Wrapper | Reset SSH connection context (ensure new ansible_host is used)"
+      meta: reset_connection
+
+- name: Phase 3 gate | Skip full scan if cloud-offline
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: "Wrapper | Gate: end_host if wrapper_skip_full_scan is set"
+      when: wrapper_skip_full_scan | default(false)
+      meta: end_host
+
+
+- import_playbook: sot-updater.yml
+
+
+- name: Phase 4 | Update action fields after sot-updater
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Scheduler | Set action_state done"
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'done'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: "Scheduler | Set action_last sot-updater-scheduler"
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_last',
+            'task_result': 'sot-updater-scheduler'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/update-indoor-222r6820-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor222-r6820.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-223r6828-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor223-r6828.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-224r6820-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor222-r6820.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-224r6831-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor224-r6831.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-225r6848-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor225-r6848.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoor-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoor6-stable.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoorbackup223r6828-scheduler.yml

@@ -0,0 +1,97 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoorbackup223r6828.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoorbackup224r6831-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoorbackup224r6831.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-indoorbackup225r6848-scheduler.yml

@@ -0,0 +1,100 @@
+# update-indoor-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-indoor6-stable.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: update-indoorbackup225r6848.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-outdoor-bootenv.yml

@@ -0,0 +1,791 @@
+- name: Fix outdoor bootenv safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # Future post-bootenv check scheduling (kept disabled for now)
+    afterbootenv_hold_exchange: "{{ lookup('env','AFTERBOOTENV_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterbootenv_routing_key: "{{ lookup('env','AFTERBOOTENV_ROUTING_KEY') | default('deviceconfig', true) }}"
+    afterbootenv_hold_queue: "{{ lookup('env','AFTERBOOTENV_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+    afterbootenv_check_enabled: false
+
+    bootenv_filename: "fox100_bootenv.bin"
+    bootenv_sha256: "324337e20b0a2d8048c359bfa2a1b8dffd6b1a28eab260143dd895ca39c034aa"
+    bootenv_path: "/tmp/{{ bootenv_filename }}"
+    bootenv_mtd_device: "/dev/mtdblock8"
+    bootenv_expected_size: "65536"
+
+    # Helper computed vars
+    bootenv_name: "{{ bootenv_filename | regex_replace('\\.bin$', '') }}"
+    bootenv_marker: "/tmp/bootenv_fixed_{{ bootenv_name }}"
+    bootenv_lock_marker: "/tmp/bootenv_fix_inprogress_{{ bootenv_name }}"
+    firmware_guard_marker: "/tmp/prepared_for_{{ bootenv_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping bootenv steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Bootenv fix cancelled because a prepared marker is already present; " ~
+                  "expected marker " ~ firmware_guard_marker ~ ". Skipping bootenv steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    - name: Check if bootenv lock marker already exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e '{{ bootenv_lock_marker }}' ] && echo PRESENT || echo ABSENT"
+      register: bootenv_lock_scan
+      changed_when: false
+
+    - name: Debug bootenv lock marker presence
+      ansible.builtin.debug:
+        msg: "bootenv_lock_marker={{ bootenv_lock_scan.stdout | trim }}"
+
+    - name: Journal bootenv fix already in progress, skipping
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Bootenv fix cancelled because lock marker already exists; " ~
+                  "marker " ~ bootenv_lock_marker ~ ". Another process may be running."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_lock_present
+      changed_when: (rmq_journal_lock_present.json is defined) and (rmq_journal_lock_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_lock_present.status != 200) or
+        (rmq_journal_lock_present.json is not defined) or
+        (not (rmq_journal_lock_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
+
+    - name: End play for this host (bootenv lock already present)
+      ansible.builtin.meta: end_host
+      when: (bootenv_lock_scan.stdout | trim) == 'PRESENT'
+
+    - name: Create bootenv lock marker
+      ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_lock_marker }}'"
+      changed_when: true
+
+    - name: Bootenv fix main block
+      block:
+        - name: Count fw_printenv lines before bootenv write
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count_before
+          changed_when: false
+
+        - name: Debug fw_printenv line count before bootenv write
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines_before={{ env_line_count_before.stdout | trim }}"
+
+        - name: Note if bootloader environment looks invalid before bootenv write (<7 lines)
+          ansible.builtin.debug:
+            msg: "Proceeding with bootenv repair even though fw_printenv returned only {{ env_line_count_before.stdout | trim }} lines (<7) before bootenv write."
+          when: (env_line_count_before.stdout | trim | int) < 7
+
+        - name: Check if bootenv image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ bootenv_path }}' ] && echo OK || echo MISSING"
+          register: bootenv_exists
+          changed_when: false
+
+        - name: Upload bootenv to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ bootenv_src_local | default("/opt/containers/ansible-worker/files/fox100_bootenv.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ bootenv_path }}"
+          delegate_to: localhost
+          when: bootenv_exists.stdout is not defined or (bootenv_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check bootenv presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ bootenv_path }}' && echo OK || echo MISSING"
+          register: bootenv_exists2
+          changed_when: false
+          failed_when: (bootenv_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded bootenv image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ bootenv_path }}' | awk '{print $1}'"
+          register: bootenv_sha_out
+          changed_when: false
+
+        - name: Verify bootenv sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ bootenv_path }}. Got {{ bootenv_sha_out.stdout | trim }}, expected {{ bootenv_sha256 }}"
+          when: (bootenv_sha_out.stdout | trim) != (bootenv_sha256 | trim)
+
+        - name: Bootenv sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ bootenv_sha_out.stdout | trim }}"
+
+        - name: Read bootenv image size on target
+          ansible.builtin.raw: "{{ pathprefix }} wc -c < '{{ bootenv_path }}'"
+          register: bootenv_size_out
+          changed_when: false
+
+        - name: Verify bootenv image size matches expected
+          ansible.builtin.fail:
+            msg: "Bootenv size mismatch for {{ bootenv_path }}. Got {{ bootenv_size_out.stdout | trim }}, expected {{ bootenv_expected_size }}"
+          when: (bootenv_size_out.stdout | trim) != (bootenv_expected_size | string | trim)
+
+        - name: Bootenv size verification debug
+          ansible.builtin.debug:
+            msg:
+              - "bootenv_size={{ bootenv_size_out.stdout | trim }}"
+
+        - name: Write bootenv to flash with dd
+          ansible.builtin.raw: "{{ pathprefix }} dd if='{{ bootenv_path }}' of={{ bootenv_mtd_device }}"
+          register: bootenv_dd
+          changed_when: true
+          failed_when: bootenv_dd.rc != 0
+
+        - name: Debug dd output (bootenv)
+          ansible.builtin.debug:
+            msg:
+              - "dd.rc={{ bootenv_dd.rc | default('NA') }}"
+              - "dd.stdout={{ (bootenv_dd.stdout | default('')) | trim }}"
+              - "dd.stderr={{ (bootenv_dd.stderr | default('')) | trim }}"
+
+        - name: Run sync after bootenv write
+          ansible.builtin.raw: "{{ pathprefix }} sync"
+          register: bootenv_sync
+          changed_when: true
+          failed_when: bootenv_sync.rc != 0
+
+        - name: Debug sync output (bootenv)
+          ansible.builtin.debug:
+            msg:
+              - "sync.rc={{ bootenv_sync.rc | default('NA') }}"
+              - "sync.stdout={{ (bootenv_sync.stdout | default('')) | trim }}"
+              - "sync.stderr={{ (bootenv_sync.stderr | default('')) | trim }}"
+
+        - name: Set bootenv write success flag
+          ansible.builtin.set_fact:
+            _bootenv_write_success: "{{ (bootenv_dd.rc | default(1)) == 0 and (bootenv_sync.rc | default(1)) == 0 }}"
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ firmware_guard_marker }}'"
+          changed_when: true
+
+        - name: Create bootenv marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ bootenv_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target bootenv)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "bootenv_outdoor_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ bootenv_filename }}"
+          when: _bootenv_write_success | bool
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Bootenv target file: {{ bootenv_filename }}"
+              - "Bootenv SHA256: OK ({{ bootenv_sha_out.stdout | trim }})"
+              - "Bootenv size: {{ bootenv_size_out.stdout | trim }} bytes"
+              - "fw_printenv lines before write: {{ env_line_count_before.stdout | trim }}"
+              - "dd: OK"
+              - "sync: OK"
+              - "Marker: {{ bootenv_marker }}"
+              - "Guard marker: {{ firmware_guard_marker }}"
+
+        - name: Compute reboot delay in seconds (default immediate when rebootin missing)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | default(0) | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Bootenv fix complete using {{ bootenv_filename }} written to {{ bootenv_mtd_device }}.
+                Original fw_printenv line count was {{ env_line_count_before.stdout | trim }}.
+                SHA256 OK {{ bootenv_sha_out.stdout | trim }}.
+                Size {{ bootenv_size_out.stdout | trim }} bytes.
+                Marker {{ bootenv_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate bootenv change.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate bootenv change.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Init after-bootenv scheduling vars
+          ansible.builtin.set_fact:
+            ab_attempt: 1
+            ab_max_attempts: 3
+            ab_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Build after-bootenv check payload (attempt 1)
+          ansible.builtin.set_fact:
+            ab_attempt: 1
+            ab_delay_sec: "{{ ab_delay_sec | default(300) }}"
+            ab_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            ab_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterbootenv_payload:
+              task_name: "afterbootenv_outdoor_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_bootenv_file: "{{ bootenv_filename }}"
+              original_fw_printenv_lines: "{{ env_line_count_before.stdout | trim }}"
+              attempt: "{{ ab_attempt | default(1) }}"
+              max_attempts: "{{ ab_max_attempts | default(3) }}"
+              current_delay_sec: "{{ ab_delay_sec | default(300) }}"
+              correlation_id: "{{ ab_correlation_id }}"
+              original_emitted_at: "{{ ab_original_emitted_at }}"
+              schema_version: 1
+              is_run_by: "{{ is_run_by_effective }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (ab_delay_sec | int) * 1000 }}"
+          when: afterbootenv_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-bootenv payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterbootenv_payload={{ afterbootenv_payload | to_nice_json }}"
+              - "ab_delay_sec={{ ab_delay_sec }}"
+              - "afterbootenv_check_enabled={{ afterbootenv_check_enabled }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+        - name: Bandaid | Pause before Publish delayed after-bootenv check to holding exchange
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when:
+            - afterbootenv_payload is defined
+            - afterbootenv_check_enabled | bool
+
+        - name: Publish delayed after-bootenv check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (ab_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterbootenv_routing_key }}"
+              payload: "{{ afterbootenv_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterbootenv_resp
+          changed_when: (rmq_afterbootenv_resp.json is defined) and (rmq_afterbootenv_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterbootenv_resp.status != 200) or
+            (rmq_afterbootenv_resp.json is not defined)
+          when:
+            - afterbootenv_payload is defined
+            - afterbootenv_check_enabled | bool
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state waiting (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'waiting'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Compute action_restart_timestamp (unix seconds) when rebootin == 0
+          ansible.builtin.set_fact:
+            action_restart_timestamp: "{{ lookup('pipe','date -u +%s') | int }}"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - reboot_seconds is defined
+            - (reboot_seconds | int) == 0
+
+        - name: Scheduler | Compute action_restart_timestamp (unix seconds) when reboot scheduled
+          ansible.builtin.set_fact:
+            action_restart_timestamp: "{{ (lookup('pipe','date -u +%s') | int) + (reboot_seconds | int) }}"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - reboot_seconds is defined
+            - (reboot_seconds | int) > 0
+
+        - name: Scheduler | Debug computed action_restart_timestamp
+          ansible.builtin.debug:
+            msg: "action_restart_timestamp={{ action_restart_timestamp }} (reboot_seconds={{ reboot_seconds | int }})"
+          when:
+            - is_run_by_effective == 'scheduler'
+            - action_restart_timestamp is defined
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_restart_timestamp custom field
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_restart_timestamp',
+                'task_result': (action_restart_timestamp | string)
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when:
+            - is_run_by_effective == 'scheduler'
+            - action_restart_timestamp is defined
+
+        - name: Remove bootenv lock marker after success
+          ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
+          changed_when: true
+
+      rescue:
+        - name: Remove bootenv lock marker after failure
+          ansible.builtin.raw: "{{ pathprefix }} rm -f '{{ bootenv_lock_marker }}'"
+          changed_when: true
+          ignore_errors: true
+
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Bootenv fix aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (bootenv failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (bootenv failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (bootenv failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-outdoorbackup-scheduler.yml

@@ -0,0 +1,156 @@
+# update-outdoorbackup-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-outdoorbackup224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+      
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-outdoorbackup225.yml
+  vars:
+    is_run_by: "scheduler"
+
+- name: Phase 4 | Scheduled success only | Chain sot-updater-scheduler
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks:
+    - name: Scheduled success | Set action_next to sot-updater-scheduler
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success | Publish sot-updater-scheduler work message
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/update-outdoorbackup224.yml

@@ -0,0 +1,800 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.4-r9850.bin
+    firmware_sha256: "38f7dd3bb5b06a2267d7bc68e2d8351df59c2aea858d644909208e11a3970539"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read /var/run/bootbanks.json (active and backup firmux versions)
+          ansible.builtin.raw: "{{ pathprefix }} cat /var/run/bootbanks.json"
+          register: bootbanks_raw
+          changed_when: false
+
+        - name: Parse bootbanks.json
+          ansible.builtin.set_fact:
+            bootbanks_obj: "{{ bootbanks_raw.stdout | from_json }}"
+            bootbanks_active_firmux: "{{ (bootbanks_raw.stdout | from_json).active.firmux | default('unknown') }}"
+            bootbanks_backup_firmux: "{{ (bootbanks_raw.stdout | from_json).backup.firmux | default('unknown') }}"
+
+        - name: current versions (active primary and backup)
+          ansible.builtin.debug:
+            msg:
+              - "active (primary) firmux: {{ bootbanks_active_firmux }}"
+              - "backup firmux: {{ bootbanks_backup_firmux }}"
+
+        - name: Build control queue payload for skip journal (backup already prepared)
+          ansible.builtin.set_fact:
+            journal_skip_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bootbank already has {{ fw_banner_repr }} (backup.firmux={{ bootbanks_backup_firmux }}).
+                Skipping backup preparation (no flip, no reboot).
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+        - name: Bandaid | Pause before Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when: journal_skip_payload is defined
+
+        - name: Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_skip_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_skip_resp
+          changed_when: (rmq_journal_skip_resp.json is defined) and (rmq_journal_skip_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_skip_resp.status != 200) or
+            (rmq_journal_skip_resp.json is not defined) or
+            (not (rmq_journal_skip_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: journal_skip_payload is defined
+
+        - name: End play for this host (backup already prepared; backup prep not needed)
+          ansible.builtin.meta: end_host
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.4-r9850.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Cleanup prepared marker after successful backup-bank write
+          ansible.builtin.raw: "rm -f '{{ fw_marker }}'"
+          changed_when: true
+          when:
+            - fw_marker is defined
+            - fw_marker | length > 0
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_outdoorbackup"
+              task_result: "backup_prepared"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Active (primary) firmux before: {{ bootbanks_active_firmux }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Success tag selection (ONLY CHANGE) --- (ONLY CHANGE) ---
+        - name: Build control queue payload for update-backup-prepared (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-backup-prepared (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-backup-prepared tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bank prepared for {{ fw_banner_repr }}.
+                Active unchanged; marker {{ fw_marker }}.
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state done (success) (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'done'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (update failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (update failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (update failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-outdoorbackup225.yml

@@ -0,0 +1,800 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.5-r9858.bin
+    firmware_sha256: "fd28e4ebef67f12a70152b9261bc7f1bdf8890bda9548d670dfc7bad98f98350"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read /var/run/bootbanks.json (active and backup firmux versions)
+          ansible.builtin.raw: "{{ pathprefix }} cat /var/run/bootbanks.json"
+          register: bootbanks_raw
+          changed_when: false
+
+        - name: Parse bootbanks.json
+          ansible.builtin.set_fact:
+            bootbanks_obj: "{{ bootbanks_raw.stdout | from_json }}"
+            bootbanks_active_firmux: "{{ (bootbanks_raw.stdout | from_json).active.firmux | default('unknown') }}"
+            bootbanks_backup_firmux: "{{ (bootbanks_raw.stdout | from_json).backup.firmux | default('unknown') }}"
+
+        - name: current versions (active primary and backup)
+          ansible.builtin.debug:
+            msg:
+              - "active (primary) firmux: {{ bootbanks_active_firmux }}"
+              - "backup firmux: {{ bootbanks_backup_firmux }}"
+
+        - name: Build control queue payload for skip journal (backup already prepared)
+          ansible.builtin.set_fact:
+            journal_skip_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bootbank already has {{ fw_banner_repr }} (backup.firmux={{ bootbanks_backup_firmux }}).
+                Skipping backup preparation (no flip, no reboot).
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+        - name: Bandaid | Pause before Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when: journal_skip_payload is defined
+
+        - name: Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_skip_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_skip_resp
+          changed_when: (rmq_journal_skip_resp.json is defined) and (rmq_journal_skip_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_skip_resp.status != 200) or
+            (rmq_journal_skip_resp.json is not defined) or
+            (not (rmq_journal_skip_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: journal_skip_payload is defined
+
+        - name: End play for this host (backup already prepared; backup prep not needed)
+          ansible.builtin.meta: end_host
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.5-r9858.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Cleanup prepared marker after successful backup-bank write
+          ansible.builtin.raw: "rm -f '{{ fw_marker }}'"
+          changed_when: true
+          when:
+            - fw_marker is defined
+            - fw_marker | length > 0
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_outdoorbackup"
+              task_result: "backup_prepared"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Active (primary) firmux before: {{ bootbanks_active_firmux }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Success tag selection (ONLY CHANGE) --- (ONLY CHANGE) ---
+        - name: Build control queue payload for update-backup-prepared (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-backup-prepared (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-backup-prepared tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bank prepared for {{ fw_banner_repr }}.
+                Active unchanged; marker {{ fw_marker }}.
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state done (success) (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'done'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (update failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (update failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (update failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-outdoorbackup226-scheduler.yml

@@ -0,0 +1,156 @@
+# update-outdoorbackup-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-outdoorbackup224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+      
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-outdoorbackup226.yml
+  vars:
+    is_run_by: "scheduler"
+
+- name: Phase 4 | Scheduled success only | Chain sot-updater-scheduler
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks:
+    - name: Scheduled success | Set action_next to sot-updater-scheduler
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Scheduled success | Publish sot-updater-scheduler work message
+      when:
+        - hostvars[inventory_hostname].journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/update-outdoorbackup226.yml

@@ -0,0 +1,800 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # Invocation context (default: manual; scheduler wrapper sets is_run_by=scheduler)
+    is_run_by_effective: "{{ is_run_by | default('manual') }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.6-r9926.bin
+    firmware_sha256: "7a3cf094ec33e272468f6680409c9acb8eff662b1b1ef020c307852d1263221c"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (hostname preflight)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (hostname preflight)"
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    - name: Scheduler context | Debug is_run_by
+      ansible.builtin.debug:
+        msg: "is_run_by={{ is_run_by_effective }}"
+
+    - name: Scheduler | Publish action_state inprogress (scheduler-run)
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'action_state',
+            'task_result': 'inprogress'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+      delegate_to: localhost
+      when: is_run_by_effective == 'scheduler'
+
+    - name: Scheduler | Debug action_state inprogress published
+      ansible.builtin.debug:
+        msg: "scheduler-run detected; published action_state=inprogress"
+      when: is_run_by_effective == 'scheduler'
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read /var/run/bootbanks.json (active and backup firmux versions)
+          ansible.builtin.raw: "{{ pathprefix }} cat /var/run/bootbanks.json"
+          register: bootbanks_raw
+          changed_when: false
+
+        - name: Parse bootbanks.json
+          ansible.builtin.set_fact:
+            bootbanks_obj: "{{ bootbanks_raw.stdout | from_json }}"
+            bootbanks_active_firmux: "{{ (bootbanks_raw.stdout | from_json).active.firmux | default('unknown') }}"
+            bootbanks_backup_firmux: "{{ (bootbanks_raw.stdout | from_json).backup.firmux | default('unknown') }}"
+
+        - name: current versions (active primary and backup)
+          ansible.builtin.debug:
+            msg:
+              - "active (primary) firmux: {{ bootbanks_active_firmux }}"
+              - "backup firmux: {{ bootbanks_backup_firmux }}"
+
+        - name: Build control queue payload for skip journal (backup already prepared)
+          ansible.builtin.set_fact:
+            journal_skip_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bootbank already has {{ fw_banner_repr }} (backup.firmux={{ bootbanks_backup_firmux }}).
+                Skipping backup preparation (no flip, no reboot).
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+        - name: Bandaid | Pause before Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+          when: journal_skip_payload is defined
+
+        - name: Publish skip journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_skip_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_skip_resp
+          changed_when: (rmq_journal_skip_resp.json is defined) and (rmq_journal_skip_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_skip_resp.status != 200) or
+            (rmq_journal_skip_resp.json is not defined) or
+            (not (rmq_journal_skip_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: journal_skip_payload is defined
+
+        - name: End play for this host (backup already prepared; backup prep not needed)
+          ansible.builtin.meta: end_host
+          when: (bootbanks_backup_firmux | trim) == (fw_banner_repr | trim)
+
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.6-r9926.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Cleanup prepared marker after successful backup-bank write
+          ansible.builtin.raw: "rm -f '{{ fw_marker }}'"
+          changed_when: true
+          when:
+            - fw_marker is defined
+            - fw_marker | length > 0
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_outdoorbackup"
+              task_result: "backup_prepared"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Bandaid | Pause before Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Active (primary) firmux before: {{ bootbanks_active_firmux }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Success tag selection (ONLY CHANGE) --- (ONLY CHANGE) ---
+        - name: Build control queue payload for update-backup-prepared (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-backup-prepared (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-backup-prepared tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-backup-prepared"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Bandaid | Pause before Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-backup-prepared tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Backup bank prepared for {{ fw_banner_repr }}.
+                Active unchanged; marker {{ fw_marker }}.
+          when: nbq2_payload_obj is defined
+
+        - name: Bandaid | Pause before Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state done (success) (success)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'done'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Bandaid | Pause before Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Bandaid | Pause before Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.pause:
+            seconds: 1
+          delegate_to: localhost
+          changed_when: false
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Scheduler | Publish action_state failed (update failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ {
+                'inscope_device': (ansible_hostname | default(inventory_hostname)),
+                'task_name': 'custom_field_set',
+                'task_add1': 'action_state',
+                'task_result': 'failed'
+              } | to_json }}"
+              payload_encoding: "string"
+          changed_when: false
+          delegate_to: localhost
+          when: is_run_by_effective == 'scheduler'
+
+        - name: Scheduler | Debug action_state failed published (update failure)
+          ansible.builtin.debug:
+            msg: "scheduler-run detected; published action_state=failed (update failure)"
+          when: is_run_by_effective == 'scheduler'

files/ansible-playbooks/update-reboot-scheduler.yml

@@ -0,0 +1,97 @@
+# update-reboot-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-rebootin224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-rebootin224.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-reboot.yml

@@ -1,4 +1,4 @@
 ---
 # update-reboot.yml — thin wrapper, no var forwarding. 
 # Delegates entirely to the unified updater.
-- import_playbook: update-rebootin223.yml
+- import_playbook: update-rebootin224.yml

files/ansible-playbooks/update-reboot225-scheduler.yml

@@ -0,0 +1,100 @@
+# update-reboot-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-rebootin224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-rebootin225.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/update-reboot226-scheduler.yml

@@ -0,0 +1,100 @@
+# update-reboot-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run update-rebootin224.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Continue with inventory IP if nb_ip not available"
+      debug:
+        msg: "Wrapper preflight did not produce nb_ip, keeping current inventory ansible_host. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      when: nb_ip_ok
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+      when: nb_ip_ok
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+      
+- import_playbook: update-rebootin226.yml
+  vars:
+    is_run_by: "scheduler"

files/ansible-playbooks/wifidebug-scheduler.yml

@@ -0,0 +1,244 @@
+# wifidebug-scheduler.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run wifidebug17.yml
+
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('([0-9]{1,3}(?:\\.[0-9]{1,3}){3})') | default('') }}"
+      set_fact:
+        nb_ip: "{{ _ip_raw }}"
+        nb_ip_ok: "{{ (_ip_raw | length) > 0 }}"
+        nb_ip_err: "{{ _fail_line }}"
+      changed_when: false
+
+    - name: "Wrapper | Abort if nb_ip not available"
+      fail:
+        msg: "Wrapper preflight did not produce nb_ip. FAIL line: {{ nb_ip_err | default('') }}"
+      when: not nb_ip_ok
+
+    - name: "Wrapper | Debug before ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(before)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+          - "nb_ip={{ nb_ip | default('UNSET') }}"
+      changed_when: false
+
+    - name: "Wrapper | Update ansible_host to nb_ip"
+      set_fact:
+        ansible_host: "{{ nb_ip }}"
+      changed_when: false
+
+    - name: "Wrapper | Debug after ansible_host override"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(after)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+    - name: "Wrapper | Reset connection after ansible_host change"
+      meta: reset_connection
+
+    - name: "Wrapper | Debug after reset_connection"
+      debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "ansible_host(post_reset)={{ ansible_host | default('UNSET') }}"
+          - "ansible_port={{ ansible_port | default('22') }}"
+      changed_when: false
+
+
+    - name: "Wrapper | Pause 2s before next playbook"
+      pause:
+        seconds: 2
+      changed_when: false
+
+
+- import_playbook: wifidebug17.yml
+  vars:
+    is_run_by: "scheduler"
+
+
+- name: Phase 4 | NetBox wrapup and chaining for wifidebug
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+  tasks:
+    - name: Wrapper compute wifidebug outcome
+      set_fact:
+        inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
+        wifidebug_ok: "{{ wifidebug_success | default(false) | bool }}"
+        wifidebug_status: "{{ wifidebug_result_status | default('UNKNOWN') }}"
+      changed_when: false
+
+    - name: Wrapper pause before clear action_next_timestamp
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper clear action_next_timestamp
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next_timestamp', 'task_result': '' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before set action_state
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_state
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_state', 'task_result': (wifidebug_ok | ternary('done','failed')) } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before chaining to sot updater
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper set action_next to sot-updater-scheduler
+      when: wifidebug_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'custom_field_set', 'task_add1': 'action_next', 'task_result': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper publish sot-updater-scheduler work message
+      when: wifidebug_ok
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig' | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "deviceconfig"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'sot-updater-scheduler' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Wrapper pause before final journal
+      ansible.builtin.pause:
+        seconds: 2
+      delegate_to: localhost
+      changed_when: false
+
+    - name: Wrapper final journal report
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ { 'inscope_device': inscope_device_name, 'task_name': 'journal_add', 'task_result': (wifidebug_ok | ternary('wifidebug: successfully deployed','wifidebug: deployment failed')) ~ ' (' ~ wifidebug_status ~ ')' } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false

files/ansible-playbooks/wifidebug17.yml

@@ -387,6 +387,12 @@
           set_fact:
             inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
 
+        - name: Expose outcome facts for scheduler wrapper
+          set_fact:
+            wifidebug_result_status: "{{ result_status | default('UNKNOWN') }}"
+            wifidebug_success: "{{ (result_status | default('')) in ['SUCCESS_DEPLOYED','SUCCESS_NO_CHANGE'] }}"
+          changed_when: false
+
         # (a) Set custom field wifidebug -> v15 on control queue
 
         - name: Build custom-field payload (wifidebug -> version)

files/files/connstats-crond-root

@@ -0,0 +1 @@
+*/10 * * * * /root/connstats.sh --debug --always-find-offset  >/dev/null 2>&1

files/files/connstats.sh

@@ -0,0 +1,678 @@
+#!/bin/sh
+# connstats.sh
+# Log-derived Wi-Fi connection counters per run (intended for cron every 10 minutes).
+# Writes one summary line per run to /root/connstats.log
+# Optional: --debug for chatty troubleshooting output in the same log.
+# Optional: --always-find-offset to calibrate syslog header time via a logger marker
+#           and (on fresh start / rotation / truncation) build a true last-10-min slice.
+
+SCRIPTVERSION=v1
+
+LOG_SRC="/var/log/messages"
+OUT_LOG="/root/connstats.log"
+STATE="/tmp/connstats.state"
+TMPDIR="/tmp"
+
+# How many tail lines to scan when we have no state (first run / after reboot / rotation)
+BOOTSTRAP_BACKFILL_LINES=2000
+
+# Used only with --always-find-offset during bootstrap-like runs
+BOOTSTRAP_TIME_WINDOW_SEC=600
+BOOTSTRAP_TIME_TAIL_LINES=10000
+
+DEBUG=0
+ALWAYS_FIND_OFFSET=0
+
+for arg in "$@"; do
+  case "$arg" in
+    --debug) DEBUG=1 ;;
+    --always-find-offset) ALWAYS_FIND_OFFSET=1 ;;
+    -h|--help)
+      echo "Usage: $0 [--debug] [--always-find-offset]"
+      exit 0
+      ;;
+    *)
+      echo "Unknown arg: $arg" >&2
+      echo "Usage: $0 [--debug] [--always-find-offset]" >&2
+      exit 2
+      ;;
+  esac
+done
+
+umask 077
+
+ts_iso() {
+  # ISO-ish timestamp; works on BusyBox and GNU date
+  date "+%Y-%m-%dT%H:%M:%S%z"
+}
+
+log() {
+  # Always append to OUT_LOG
+  echo "[$(ts_iso)] $*" >> "$OUT_LOG"
+}
+
+# Generate a short random token (8 chars) for marker grep
+rand_token() {
+  if [ -r /dev/urandom ]; then
+    tr -dc 'a-z0-9' < /dev/urandom 2>/dev/null | head -c 8
+  else
+    echo "$(date -u +%s 2>/dev/null)$$" | tr -dc 'a-z0-9' | tail -c 8
+  fi
+}
+
+# Parse syslog header timestamp (Mon DD HH:MM:SS) into epoch seconds (UTC) using awk mktime()
+# Expects typical line like: "<13> Jan 31 12:31:00 root[-] [notice]: message"
+syslog_header_to_epoch_utc() {
+  echo "$1" | TZ=UTC awk '
+    function mon2num(m) {
+      if (m=="Jan") return 1
+      if (m=="Feb") return 2
+      if (m=="Mar") return 3
+      if (m=="Apr") return 4
+      if (m=="May") return 5
+      if (m=="Jun") return 6
+      if (m=="Jul") return 7
+      if (m=="Aug") return 8
+      if (m=="Sep") return 9
+      if (m=="Oct") return 10
+      if (m=="Nov") return 11
+      if (m=="Dec") return 12
+      return 0
+    }
+    {
+      mon=$2; day=$3; tod=$4
+      year=strftime("%Y")
+      m=mon2num(mon)
+      if (m==0) { print ""; exit 1 }
+      split(tod, t, ":")
+      if (length(t) != 3) { print ""; exit 1 }
+      hh=t[1]+0; mm=t[2]+0; ss=t[3]+0
+      d=day+0
+      print mktime(sprintf("%d %02d %02d %02d %02d %02d", year, m, d, hh, mm, ss))
+    }
+  '
+}
+
+# Compute offset_sec = payload_epoch_utc - syslog_header_epoch_utc
+# Returns offset seconds on stdout (blank on failure)
+find_time_offset_sec() {
+  token="$(rand_token)"
+  now_epoch="$(date -u +%s 2>/dev/null | tr -d ' ')"
+  [ -z "$now_epoch" ] && now_epoch=0
+
+  logger "connstat calculation started. issueing time marker $token epoch=$now_epoch" 2>/dev/null
+
+  found_line=""
+  i=0
+  while [ $i -lt 3 ]; do
+    found_line="$(tail -n 300 "$LOG_SRC" 2>/dev/null | grep "$token" | tail -n 1)"
+    [ -n "$found_line" ] && break
+    sleep 1
+    i=$((i + 1))
+  done
+
+  [ -z "$found_line" ] && { echo ""; return; }
+
+  hdr_epoch="$(syslog_header_to_epoch_utc "$found_line" | tr -d ' ')"
+  [ -z "$hdr_epoch" ] && { echo ""; return; }
+
+  payload_epoch="$(echo "$found_line" | awk '{
+    for (i=1; i<=NF; i++) {
+      if ($i ~ /^epoch=[0-9]+$/) { sub(/^epoch=/,"",$i); print $i; exit }
+    }
+  }' | tr -d ' ')"
+  [ -z "$payload_epoch" ] && payload_epoch="$now_epoch"
+
+  echo $((payload_epoch - hdr_epoch))
+}
+
+# Basic sanity checks
+if [ ! -r "$LOG_SRC" ]; then
+  log "ERROR log_src_unreadable path=$LOG_SRC"
+  exit 1
+fi
+
+# Get current inode + line count (line-based incremental read is most portable)
+cur_inode="$(ls -i "$LOG_SRC" 2>/dev/null | awk '{print $1}')"
+cur_lines="$(wc -l < "$LOG_SRC" 2>/dev/null | tr -d ' ')"
+
+if [ -z "$cur_inode" ] || [ -z "$cur_lines" ]; then
+  log "ERROR cannot_stat_log inode='$cur_inode' lines='$cur_lines'"
+  exit 1
+fi
+
+last_inode=""
+last_line=""
+
+if [ -f "$STATE" ]; then
+  # STATE format:
+  # inode=<num>
+  # line=<num>
+  last_inode="$(grep '^inode=' "$STATE" 2>/dev/null | head -n1 | cut -d= -f2)"
+  last_line="$(grep '^line=' "$STATE" 2>/dev/null | head -n1 | cut -d= -f2)"
+fi
+
+# Decide where to start reading
+start_line=""
+
+backfill_start() {
+  start_line=$((cur_lines - BOOTSTRAP_BACKFILL_LINES + 1))
+  [ "$start_line" -lt 1 ] && start_line=1
+}
+
+# Extract the slice
+tmp_slice="$TMPDIR/connstats.slice.$$"
+
+# Grep patterns (hostapd transition events)
+re_auth='hostapd.*IEEE 802\.11: authenticated'
+re_assoc='hostapd.*IEEE 802\.11: associated'
+re_disassoc='hostapd.*IEEE 802\.11: disassociated'
+
+# Kernel extras for "seen_any" breadth (optional but useful)
+re_kernel_seen='kernel.*(station kicked out|station timed out)'
+
+# Are we in a bootstrap-like condition (no state OR rotation/truncation)?
+bootstrap_like=0
+if [ -z "$last_inode" ] || [ -z "$last_line" ]; then
+  bootstrap_like=1
+else
+  if [ "$cur_inode" != "$last_inode" ] || [ "$cur_lines" -lt "$last_line" ]; then
+    bootstrap_like=1
+  fi
+fi
+
+# Bootstrap handling
+if [ "$ALWAYS_FIND_OFFSET" -eq 1 ] && [ "$bootstrap_like" -eq 1 ]; then
+  offset_sec="$(find_time_offset_sec)"
+
+  if [ -n "$offset_sec" ]; then
+    now_epoch="$(date -u +%s 2>/dev/null | tr -d ' ')"
+    [ -z "$now_epoch" ] && now_epoch=0
+    cutoff_epoch=$((now_epoch - BOOTSTRAP_TIME_WINDOW_SEC))
+
+    if [ $DEBUG -eq 1 ]; then
+      log "DEBUG bootstrap_time_window_last_10m offset_sec=$offset_sec cutoff_epoch=$cutoff_epoch tail_lines=$BOOTSTRAP_TIME_TAIL_LINES"
+    fi
+
+    # Build slice = last-10-min relevant lines (hostapd transitions + kernel station kicked/timed out)
+    # using (syslog_header_epoch + offset_sec) >= cutoff_epoch
+    tail -n "$BOOTSTRAP_TIME_TAIL_LINES" "$LOG_SRC" 2>/dev/null \
+      | TZ=UTC awk -v cutoff="$cutoff_epoch" -v off="$offset_sec" '
+        function mon2num(m) {
+          if (m=="Jan") return 1
+          if (m=="Feb") return 2
+          if (m=="Mar") return 3
+          if (m=="Apr") return 4
+          if (m=="May") return 5
+          if (m=="Jun") return 6
+          if (m=="Jul") return 7
+          if (m=="Aug") return 8
+          if (m=="Sep") return 9
+          if (m=="Oct") return 10
+          if (m=="Nov") return 11
+          if (m=="Dec") return 12
+          return 0
+        }
+        function header_epoch(   mon,day,tod,year,m,hh,mm,ss,t) {
+          mon=$2; day=$3; tod=$4
+          year=strftime("%Y")
+          m=mon2num(mon)
+          if (m==0) return -1
+          split(tod, t, ":")
+          if (length(t) != 3) return -1
+          hh=t[1]+0; mm=t[2]+0; ss=t[3]+0
+          return mktime(sprintf("%d %02d %02d %02d %02d %02d", year, m, (day+0), hh, mm, ss))
+        }
+        {
+          line=$0
+          if (line ~ /hostapd.*IEEE 802\.11: (authenticated|associated|disassociated)/ ||
+              line ~ /kernel.*(station kicked out|station timed out)/) {
+            he=header_epoch()
+            if (he < 0) next
+            corr=he + off
+            if (corr >= cutoff) print line
+          }
+        }
+      ' > "$tmp_slice" 2>/dev/null
+
+    start_line="(time_window_last_10m)"
+  else
+    # Offset calc failed -> fall back to old bootstrap behavior
+    backfill_start
+    if [ $DEBUG -eq 1 ]; then
+      log "DEBUG time_offset_failed_fallback_to_backfill cur_inode=$cur_inode cur_lines=$cur_lines start_line=$start_line backfill_lines=$BOOTSTRAP_BACKFILL_LINES"
+    fi
+    sed -n "${start_line},\$p" "$LOG_SRC" > "$tmp_slice" 2>/dev/null
+  fi
+
+else
+  # Original behavior (unchanged)
+  if [ -z "$last_inode" ] || [ -z "$last_line" ]; then
+    backfill_start
+    if [ $DEBUG -eq 1 ]; then
+      log "DEBUG bootstrap_no_state_backfill cur_inode=$cur_inode cur_lines=$cur_lines start_line=$start_line backfill_lines=$BOOTSTRAP_BACKFILL_LINES"
+    fi
+  else
+    # Rotation / truncation detection
+    if [ "$cur_inode" != "$last_inode" ] || [ "$cur_lines" -lt "$last_line" ]; then
+      backfill_start
+      if [ $DEBUG -eq 1 ]; then
+        log "DEBUG bootstrap_rotation_or_truncate_backfill last_inode=$last_inode cur_inode=$cur_inode last_line=$last_line cur_lines=$cur_lines start_line=$start_line backfill_lines=$BOOTSTRAP_BACKFILL_LINES"
+      fi
+    else
+      start_line=$((last_line + 1))
+      if [ $DEBUG -eq 1 ]; then
+        log "DEBUG incremental last_line=$last_line cur_lines=$cur_lines start_line=$start_line"
+      fi
+    fi
+  fi
+
+  sed -n "${start_line},\$p" "$LOG_SRC" > "$tmp_slice" 2>/dev/null
+fi
+
+slice_lines="$(wc -l < "$tmp_slice" 2>/dev/null | tr -d ' ')"
+[ -z "$slice_lines" ] && slice_lines=0
+
+if [ $DEBUG -eq 1 ]; then
+  log "DEBUG slice_path=$tmp_slice slice_lines=$slice_lines"
+fi
+
+# Totals (MAC repeats allowed)
+auth_events_total="$(grep -E "$re_auth" "$tmp_slice" | wc -l | tr -d ' ')"
+assoc_events_total="$(grep -E "$re_assoc" "$tmp_slice" | wc -l | tr -d ' ')"
+disassoc_events_total="$(grep -E "$re_disassoc" "$tmp_slice" | wc -l | tr -d ' ')"
+
+# Extract MAC right after token "STA" (from hostapd lines)
+extract_sta_macs() {
+  awk '
+    {
+      for (i=1; i<=NF; i++) {
+        if ($i=="STA") {
+          print $(i+1);
+          break;
+        }
+      }
+    }
+  '
+}
+
+# Extract MACs in brackets: [aa:bb:cc:dd:ee:ff]
+extract_bracket_macs() {
+  grep -oE '\[[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}\]' | tr -d '[]'
+}
+
+# Count unique valid MACs from stdin
+uniq_count() {
+  grep -E '^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$' \
+  | sort -u \
+  | wc -l | tr -d ' '
+}
+
+# Unique MACs per event type (hostapd)
+unique_macs_authenticated="$(
+  grep -E "$re_auth" "$tmp_slice" \
+  | extract_sta_macs \
+  | uniq_count
+)"
+
+unique_macs_associated="$(
+  grep -E "$re_assoc" "$tmp_slice" \
+  | extract_sta_macs \
+  | uniq_count
+)"
+
+unique_macs_disassociated="$(
+  grep -E "$re_disassoc" "$tmp_slice" \
+  | extract_sta_macs \
+  | uniq_count
+)"
+
+# Unique MACs seen in "any relevant activity" (union: auth/assoc/disassoc + selected kernel)
+unique_macs_seen_any="$(
+  (
+    grep -E 'hostapd.*IEEE 802\.11: (authenticated|associated|disassociated)' "$tmp_slice" \
+      | extract_sta_macs
+
+    grep -E "$re_kernel_seen" "$tmp_slice" \
+      | extract_bracket_macs
+  ) | uniq_count
+)"
+
+########################################################################
+# B-class metrics (ephemeral state in /tmp; reboot loss accepted)
+########################################################################
+
+SESSION_STATE="/tmp/connstats.sessions.state"
+SESSION_STATE_TMP="$TMPDIR/connstats.sessions.state.$$"
+SESSION_STATE_TTL_SEC=21600
+AUTH_ASSOC_MAX_SAMPLES=50
+
+now_epoch_b="$(date -u +%s 2>/dev/null | tr -d ' ')"
+[ -z "$now_epoch_b" ] && now_epoch_b=0
+
+# Output file (temporary, removed each run)
+B_OUT_TMP="$TMPDIR/connstats.b.out.$$"
+
+TZ=UTC awk -v state_in="$SESSION_STATE" \
+          -v state_out="$SESSION_STATE_TMP" \
+          -v now="$now_epoch_b" \
+          -v ttl="$SESSION_STATE_TTL_SEC" \
+          -v maxs="$AUTH_ASSOC_MAX_SAMPLES" '
+  function mon2num(m) {
+    if (m=="Jan") return 1
+    if (m=="Feb") return 2
+    if (m=="Mar") return 3
+    if (m=="Apr") return 4
+    if (m=="May") return 5
+    if (m=="Jun") return 6
+    if (m=="Jul") return 7
+    if (m=="Aug") return 8
+    if (m=="Sep") return 9
+    if (m=="Oct") return 10
+    if (m=="Nov") return 11
+    if (m=="Dec") return 12
+    return 0
+  }
+  function header_epoch(   mon,day,tod,year,m,hh,mm,ss,t) {
+    mon=$2; day=$3; tod=$4
+    year=strftime("%Y")
+    m=mon2num(mon)
+    if (m==0) return -1
+    split(tod, t, ":")
+    if (length(t) != 3) return -1
+    hh=t[1]+0; mm=t[2]+0; ss=t[3]+0
+    return mktime(sprintf("%d %02d %02d %02d %02d %02d", year, m, (day+0), hh, mm, ss))
+  }
+  function extract_sta_mac(   i) {
+    for (i=1; i<=NF; i++) {
+      if ($i=="STA") return $(i+1)
+    }
+    return ""
+  }
+  function mac_valid(m) {
+    return (m ~ /^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$/)
+  }
+
+  BEGIN {
+    if (state_in != "") {
+      while ((getline line < state_in) > 0) {
+        n = split(line, a, " ")
+        if (n >= 2) {
+          m=a[1]; e=a[2]+0
+          if (mac_valid(m) && e > 0) last_auth[m]=e
+        }
+      }
+      close(state_in)
+    }
+  }
+
+  {
+    line=$0
+    if (line !~ /hostapd.*IEEE 802\.11: (authenticated|associated|disassociated)/) next
+
+    e = header_epoch()
+    if (e < 0) next
+
+    mac = extract_sta_mac()
+    if (!mac_valid(mac)) next
+
+    if (line ~ /IEEE 802\.11: authenticated/) {
+      last_auth[mac]=e
+      next
+    }
+
+    if (line ~ /IEEE 802\.11: associated/) {
+      if (mac in last_auth) {
+        d = e - last_auth[mac]
+        if (d >= 0) {
+          ms = d * 1000
+          if (sample_count < maxs) {
+            if (samples == "") samples = ms
+            else samples = samples "," ms
+            sample_count++
+          }
+        }
+      }
+      next
+    }
+
+    if (line ~ /IEEE 802\.11: disassociated/) {
+      if (mac in last_auth) {
+        d = e - last_auth[mac]
+        if (d >= 0) {
+          full_sessions_total++
+          sum_len += d
+          if (d < 60) early_drop_sessions_total++
+          delete last_auth[mac]
+        }
+      }
+      next
+    }
+  }
+
+  END {
+    if (full_sessions_total > 0) avg_session_length = int((sum_len / full_sessions_total) + 0.5)
+    else avg_session_length = 0
+
+    if (state_out != "") {
+      for (m in last_auth) {
+        e = last_auth[m] + 0
+        if (e > 0 && (now <= 0 || (e >= (now - ttl)))) {
+          print m " " e > state_out
+        }
+      }
+      close(state_out)
+    }
+
+    print full_sessions_total "\t" avg_session_length "\t" early_drop_sessions_total "\t" samples
+  }
+' "$tmp_slice" > "$B_OUT_TMP" 2>/dev/null
+
+# Default B fields
+full_sessions_total=0
+avg_session_length=0
+early_drop_sessions_total=0
+auth_to_assoc_time_ms_samples=""
+
+# Parse awk output
+if [ -s "$B_OUT_TMP" ]; then
+  b_out_line="$(head -n 1 "$B_OUT_TMP" 2>/dev/null)"
+  full_sessions_total="$(echo "$b_out_line" | awk -F'\t' '{print $1}' | tr -d ' ')"
+  avg_session_length="$(echo "$b_out_line" | awk -F'\t' '{print $2}' | tr -d ' ')"
+  early_drop_sessions_total="$(echo "$b_out_line" | awk -F'\t' '{print $3}' | tr -d ' ')"
+  auth_to_assoc_time_ms_samples="$(echo "$b_out_line" | awk -F'\t' '{print $4}')"
+fi
+
+rm -f "$B_OUT_TMP" 2>/dev/null
+
+[ -z "$full_sessions_total" ] && full_sessions_total=0
+[ -z "$avg_session_length" ] && avg_session_length=0
+[ -z "$early_drop_sessions_total" ] && early_drop_sessions_total=0
+
+# Move state into place (ephemeral but persistent across runs until reboot)
+if [ -f "$SESSION_STATE_TMP" ]; then
+  mv -f "$SESSION_STATE_TMP" "$SESSION_STATE" 2>/dev/null
+else
+  rm -f "$SESSION_STATE_TMP" 2>/dev/null
+fi
+
+########################################################################
+# C-class metrics (kernel kick reason codes 1..8)
+########################################################################
+
+# Only count kernel "station kicked out ..." (avoids PRS DRIVER_LOG "reason" lines)
+re_kick_reason_base='kernel.*station kicked out.*reason[[:space:]]+'
+
+reason_code_events_1="$(grep -E "${re_kick_reason_base}1([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_2="$(grep -E "${re_kick_reason_base}2([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_3="$(grep -E "${re_kick_reason_base}3([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_4="$(grep -E "${re_kick_reason_base}4([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_5="$(grep -E "${re_kick_reason_base}5([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_6="$(grep -E "${re_kick_reason_base}6([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_7="$(grep -E "${re_kick_reason_base}7([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+reason_code_events_8="$(grep -E "${re_kick_reason_base}8([^0-9]|$)" "$tmp_slice" 2>/dev/null | wc -l | tr -d ' ')"
+
+[ -z "$reason_code_events_1" ] && reason_code_events_1=0
+[ -z "$reason_code_events_2" ] && reason_code_events_2=0
+[ -z "$reason_code_events_3" ] && reason_code_events_3=0
+[ -z "$reason_code_events_4" ] && reason_code_events_4=0
+[ -z "$reason_code_events_5" ] && reason_code_events_5=0
+[ -z "$reason_code_events_6" ] && reason_code_events_6=0
+[ -z "$reason_code_events_7" ] && reason_code_events_7=0
+[ -z "$reason_code_events_8" ] && reason_code_events_8=0
+
+########################################################################
+# D-class metrics (flaps between ath0 and ath1 within the bucket)
+########################################################################
+
+D_OUT_TMP="$TMPDIR/connstats.d.out.$$"
+
+awk '
+  function extract_sta_mac(   i) {
+    for (i=1; i<=NF; i++) if ($i=="STA") return $(i+1)
+    return ""
+  }
+  function mac_valid(m) {
+    return (m ~ /^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$/)
+  }
+  function extract_iface(   i, v) {
+    for (i=1; i<=NF; i++) {
+      v = $i
+      if (v ~ /^ath[0-9]+:$/) { sub(/:$/,"",v); return v }
+    }
+    return ""
+  }
+
+  {
+    line=$0
+    if (line !~ /hostapd.*IEEE 802\.11: associated/) next
+
+    iface = extract_iface()
+    if (iface != "ath0" && iface != "ath1") next
+
+    mac = extract_sta_mac()
+    if (!mac_valid(mac)) next
+
+    if (mac in last_iface) {
+      prev = last_iface[mac]
+      if (prev != iface) {
+        if (prev=="ath0" && iface=="ath1") {
+          flap_events_0_to_1++
+          uniq_0_to_1[mac]=1
+        } else if (prev=="ath1" && iface=="ath0") {
+          flap_events_1_to_0++
+          uniq_1_to_0[mac]=1
+        }
+      }
+    }
+
+    last_iface[mac]=iface
+  }
+
+  END {
+    for (m in uniq_0_to_1) unique_0_to_1++
+    for (m in uniq_1_to_0) unique_1_to_0++
+
+    if (unique_0_to_1+0 < 0) unique_0_to_1=0
+    if (flap_events_0_to_1+0 < 0) flap_events_0_to_1=0
+    if (unique_1_to_0+0 < 0) unique_1_to_0=0
+    if (flap_events_1_to_0+0 < 0) flap_events_1_to_0=0
+
+    print unique_0_to_1 "\t" flap_events_0_to_1 "\t" unique_1_to_0 "\t" flap_events_1_to_0
+  }
+' "$tmp_slice" > "$D_OUT_TMP" 2>/dev/null
+
+unique_macs_flap_ath0_to_ath1=0
+flap_events_total_ath0_to_ath1=0
+unique_macs_flap_ath1_to_ath0=0
+flap_events_total_ath1_to_ath0=0
+
+if [ -s "$D_OUT_TMP" ]; then
+  d_out_line="$(head -n 1 "$D_OUT_TMP" 2>/dev/null)"
+  unique_macs_flap_ath0_to_ath1="$(echo "$d_out_line" | awk -F'\t' '{print $1}' | tr -d ' ')"
+  flap_events_total_ath0_to_ath1="$(echo "$d_out_line" | awk -F'\t' '{print $2}' | tr -d ' ')"
+  unique_macs_flap_ath1_to_ath0="$(echo "$d_out_line" | awk -F'\t' '{print $3}' | tr -d ' ')"
+  flap_events_total_ath1_to_ath0="$(echo "$d_out_line" | awk -F'\t' '{print $4}' | tr -d ' ')"
+fi
+
+rm -f "$D_OUT_TMP" 2>/dev/null
+
+[ -z "$unique_macs_flap_ath0_to_ath1" ] && unique_macs_flap_ath0_to_ath1=0
+[ -z "$flap_events_total_ath0_to_ath1" ] && flap_events_total_ath0_to_ath1=0
+[ -z "$unique_macs_flap_ath1_to_ath0" ] && unique_macs_flap_ath1_to_ath0=0
+[ -z "$flap_events_total_ath1_to_ath0" ] && flap_events_total_ath1_to_ath0=0
+
+########################################################################
+# Emit one summary line
+########################################################################
+log "connstats bucket_run slice_lines=$slice_lines unique_macs_seen_any=$unique_macs_seen_any unique_macs_authenticated=$unique_macs_authenticated auth_events_total=$auth_events_total unique_macs_associated=$unique_macs_associated assoc_events_total=$assoc_events_total unique_macs_disassociated=$unique_macs_disassociated disassoc_events_total=$disassoc_events_total full_sessions_total=$full_sessions_total avg_session_length=$avg_session_length early_drop_sessions_total=$early_drop_sessions_total auth_to_assoc_time_ms_samples=$auth_to_assoc_time_ms_samples reason_code_events_1=$reason_code_events_1 reason_code_events_2=$reason_code_events_2 reason_code_events_3=$reason_code_events_3 reason_code_events_4=$reason_code_events_4 reason_code_events_5=$reason_code_events_5 reason_code_events_6=$reason_code_events_6 reason_code_events_7=$reason_code_events_7 reason_code_events_8=$reason_code_events_8 unique_macs_flap_ath0_to_ath1=$unique_macs_flap_ath0_to_ath1 flap_events_total_ath0_to_ath1=$flap_events_total_ath0_to_ath1 unique_macs_flap_ath1_to_ath0=$unique_macs_flap_ath1_to_ath0 flap_events_total_ath1_to_ath0=$flap_events_total_ath1_to_ath0"
+
+if [ $DEBUG -eq 1 ]; then
+  log "DEBUG dump_auth_lines_begin"
+  grep -E "$re_auth" "$tmp_slice" | sed 's/^/DEBUG AUTH: /' >> "$OUT_LOG"
+  log "DEBUG dump_assoc_lines_begin"
+  grep -E "$re_assoc" "$tmp_slice" | sed 's/^/DEBUG ASSOC: /' >> "$OUT_LOG"
+  log "DEBUG dump_disassoc_lines_begin"
+  grep -E "$re_disassoc" "$tmp_slice" | sed 's/^/DEBUG DISASSOC: /' >> "$OUT_LOG"
+
+  log "DEBUG dump_kick_reason_lines_begin"
+  grep -E "${re_kick_reason_base}[1-8]([^0-9]|$)" "$tmp_slice" | sed 's/^/DEBUG KICK: /' >> "$OUT_LOG"
+
+  log "DEBUG dump_flap_assoc_lines_begin"
+  grep -E 'hostapd.*IEEE 802\.11: associated' "$tmp_slice" | sed 's/^/DEBUG FLAP_ASSOC: /' >> "$OUT_LOG"
+
+  log "DEBUG dump_flap_events_begin"
+  awk '
+    function extract_sta_mac(   i) { for (i=1; i<=NF; i++) if ($i=="STA") return $(i+1); return "" }
+    function mac_valid(m) { return (m ~ /^[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}$/) }
+    function extract_iface(   i, v) {
+      for (i=1; i<=NF; i++) { v=$i; if (v ~ /^ath[0-9]+:$/) { sub(/:$/,"",v); return v } }
+      return ""
+    }
+    {
+      if ($0 !~ /hostapd.*IEEE 802\.11: associated/) next
+      iface=extract_iface()
+      if (iface!="ath0" && iface!="ath1") next
+      mac=extract_sta_mac()
+      if (!mac_valid(mac)) next
+      if (mac in last_iface) {
+        prev=last_iface[mac]
+        if (prev!=iface) {
+          if ((prev=="ath0" && iface=="ath1") || (prev=="ath1" && iface=="ath0")) {
+            print "mac=" mac " from=" prev " to=" iface
+          }
+        }
+      }
+      last_iface[mac]=iface
+    }
+  ' "$tmp_slice" | sed 's/^/DEBUG FLAP: /' >> "$OUT_LOG"
+fi
+
+# ---------------------------------------------------------------------
+# Send one syslog message (wifidebug-style tag), after random delay
+# ---------------------------------------------------------------------
+
+# Hostname (as in wifidebug)
+HOSTNAME="$(cat /proc/sys/kernel/hostname 2>/dev/null)"
+[ -z "$HOSTNAME" ] && HOSTNAME="$(hostname 2>/dev/null)"
+[ -z "$HOSTNAME" ] && HOSTNAME="unknown"
+
+# Random delay 3..20 seconds (right before sending)
+delay="$(hexdump -n2 -e '/2 "%u"' /dev/urandom 2>/dev/null)"
+case "$delay" in (''|*[!0-9]*) delay=0;; esac
+delay=$(( (delay % 18) + 3 ))
+sleep "$delay"
+
+# Exact stats payload (same fields as file line, no extra prefixes)
+MSG="slice_lines=$slice_lines unique_macs_seen_any=$unique_macs_seen_any unique_macs_authenticated=$unique_macs_authenticated auth_events_total=$auth_events_total unique_macs_associated=$unique_macs_associated assoc_events_total=$assoc_events_total unique_macs_disassociated=$unique_macs_disassociated disassoc_events_total=$disassoc_events_total full_sessions_total=$full_sessions_total avg_session_length=$avg_session_length early_drop_sessions_total=$early_drop_sessions_total auth_to_assoc_time_ms_samples=$auth_to_assoc_time_ms_samples reason_code_events_1=$reason_code_events_1 reason_code_events_2=$reason_code_events_2 reason_code_events_3=$reason_code_events_3 reason_code_events_4=$reason_code_events_4 reason_code_events_5=$reason_code_events_5 reason_code_events_6=$reason_code_events_6 reason_code_events_7=$reason_code_events_7 reason_code_events_8=$reason_code_events_8 unique_macs_flap_ath0_to_ath1=$unique_macs_flap_ath0_to_ath1 flap_events_total_ath0_to_ath1=$flap_events_total_ath0_to_ath1 unique_macs_flap_ath1_to_ath0=$unique_macs_flap_ath1_to_ath0 flap_events_total_ath1_to_ath0=$flap_events_total_ath1_to_ath0"
+
+logger -t "connstats|${HOSTNAME}" "$MSG" 2>/dev/null
+
+# Update state to current end-of-file line count and inode
+{
+  echo "inode=$cur_inode"
+  echo "line=$cur_lines"
+} > "$STATE"
+
+rm -f "$tmp_slice" 2>/dev/null
+exit 0

files/nb_onedevice_update-old.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""
+nb_sync_one_device.py
+
+Update a single NetBox device from Cloud by hostname — ONLY if device is online.
+
+Behavior
+- Liveness gate via /v2/devices/{cloud_id}/liveness (no changes if offline).
+- Updates NetBox fields from Cloud:
+  custom_fields.fw_version ← firmwareVersion
+  custom_fields.nodeName   ← nodeName
+  custom_fields.sectorName ← sectorName
+  custom_fields.smallCellName ← smallCellName
+  serial ← serialNumber
+- IP handling:
+  If Cloud ipAddress is valid (not None/""/"0.0.0.0"):
+    ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
+    then PRUNE all other IPs on this device (default).
+  If Cloud ipAddress is placeholder/invalid:
+    skip IP changes and do not prune.
+- --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
+- NEW: Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+
+Exit codes:
+  0 = success
+  1 = not found / missing data / offline
+  3 = network/HTTP error
+  4 = NetBox update error
+"""
+
+import sys
+import json
+import argparse
+from typing import Optional, Union, List, Dict
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# ------------ HARD-CODED CONFIG (per Pavel) ------------
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v2/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
+
+REQ_TIMEOUT = 30
+CHATTY = False
+
+# ------------ HTTP utilities ------------
+def _new_session() -> requests.Session:
+    s = requests.Session()
+    retries = Retry(
+        total=3, connect=3, read=3, status=3,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+        respect_retry_after_header=True,
+    )
+    adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
+    s.mount('http://', adapter)
+    s.mount('https://', adapter)
+    s.headers['Accept'] = 'application/json'
+    return s
+
+S_NB = _new_session()
+S_CL = _new_session()
+
+# ------------ Logging / status helpers ------------
+def log(msg: str):
+    if CHATTY:
+        print(msg, file=sys.stderr)
+
+def die(code: int, msg: str):
+    # Single-line result: stdout on success, stderr on failure
+    print(msg, file=sys.stdout if code == 0 else sys.stderr)
+    raise SystemExit(code)
+
+# ------------ NetBox API ------------
+class NetBox:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip('/')
+        self.token = token
+
+    def _h(self):
+        return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def get_device_by_name(self, name: str) -> Optional[dict]:
+        log(f"NB: lookup device by name {name}")
+        r = S_NB.get(self._url("/api/dcim/devices/"), headers=self._h(),
+                     params={"name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET devices name={name} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def get_device(self, dev_id: int) -> dict:
+        log(f"NB: fetch device id={dev_id}")
+        r = S_NB.get(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET device id={dev_id} HTTP={r.status_code}")
+        return r.json()
+
+    def patch_device(self, dev_id: int, patch: dict) -> None:
+        if not patch:
+            log("NB: no device patch needed")
+            return
+        log(f"NB: patch device id={dev_id} keys={list(patch.keys())}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps(patch), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL patch device dev={dev_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                     params={"device_id": dev_id, "name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET interfaces device_id={dev_id} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0]["id"] if res else None
+
+    def ensure_eth0(self, dev_id: int) -> int:
+        ifid = self.get_iface_id(dev_id, "eth0")
+        log(f"NB: ensure eth0 (current id={ifid})")
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = S_NB.post(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        if r.status_code == 400:
+            # race: read again
+            ifid = self.get_iface_id(dev_id, "eth0")
+            if ifid:
+                return ifid
+        die(4, f"FAIL create eth0 HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_ip_by_addr(self, addr: str) -> Optional[dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"address": f"{addr}/32"}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET ip-addresses addr={addr} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def create_ip_for_iface(self, addr: str, iface_id: int) -> int:
+        payload = {
+            "address": f"{addr}/32",
+            "status": "active",
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        }
+        r = S_NB.post(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        die(4, f"FAIL create IP {addr} HTTP={r.status_code} body={r.text[:200]}")
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
+        r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
+                       data=json.dumps({
+                           "assigned_object_type": "dcim.interface",
+                           "assigned_object_id": iface_id
+                       }),
+                       timeout=REQ_TIMEOUT)
+        return 200 <= r.status_code < 300
+
+    def device_set_primary_ip4(self, dev_id: int, ip_id: int) -> None:
+        log(f"NB: set primary_ip4 dev={dev_id} -> ip_id={ip_id}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps({"primary_ip4": ip_id}), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL set primary_ip4 dev={dev_id} ip_id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = S_NB.get(self._url(f"/api/dcim/interfaces/{iface_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return (r.json().get("device") or {}).get("id")
+        return None
+
+    # --- Helpers for pruning ---
+    def list_device_ips(self, dev_id: int) -> List[Dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return r.json().get("results") or []
+        ips: List[Dict] = []
+        r2 = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r2.status_code == 200:
+            for iface in (r2.json().get("results") or []):
+                ifid = iface.get("id")
+                r3 = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                              params={"assigned_object_type": "dcim.interface",
+                                      "assigned_object_id": ifid, "limit": 1000}, timeout=REQ_TIMEOUT)
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get("results") or [])
+        return ips
+
+    def delete_ip(self, ip_id: int) -> None:
+        log(f"NB: delete IP id={ip_id}")
+        r = S_NB.delete(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300 or r.status_code == 204):
+            die(4, f"FAIL delete IP id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+# ------------ Cloud API ------------
+class Cloud:
+    def __init__(self, base: str, bearer: str):
+        self.base = base.rstrip("/")
+        self.bearer = bearer
+
+    def _h(self):
+        return {"Authorization": f"Bearer {self.bearer}", "Accept": "application/json"}
+
+    def device_detail(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}"
+        log(f"CL: fetch detail cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id} HTTP={r.status_code}")
+        return r.json()
+
+    def device_liveness(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}/liveness"
+        log(f"CL: fetch liveness cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
+        return r.json()
+
+# ------------ Core ------------
+def _primary_ip4_text(dev_json: dict) -> Optional[str]:
+    p = dev_json.get("primary_ip4") or {}
+    addr = p.get("address")
+    if isinstance(addr, str) and addr.endswith("/32"):
+        return addr[:-3]
+    return addr
+
+def run(hostname: str) -> None:
+    nb = NetBox(NB_URL, NB_TOKEN)
+    cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+
+    dev = nb.get_device_by_name(hostname)
+    if not dev:
+        die(1, f"FAIL {hostname} not found in NetBox")
+
+    dev_id = dev["id"]
+    dev_full = nb.get_device(dev_id)
+    cf = dev_full.get("custom_fields") or {}
+
+    # NEW: log upgrade command if present
+    upgrade_cmd = cf.get("upgrade_cmd")
+    log(f"NB: upgrade_cmd={upgrade_cmd}")
+
+    cloud_id = cf.get("cloud_id")
+    if cloud_id in (None, "", "null"):
+        die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
+
+    # 1) Liveness gate
+    live = cl.device_liveness(cloud_id)
+    if not bool(live.get("isConnected")):
+        die(1, f"FAIL {hostname} cloud_id={cloud_id} device is not online")
+
+    current_nb_ip = _primary_ip4_text(dev_full)
+
+    # 2) Cloud detail
+    d = cl.device_detail(cloud_id)
+    fw = d.get("firmwareVersion") or d.get("version")
+    ip_from_cloud = (d.get("ipAddress") or "").strip() if isinstance(d.get("ipAddress"), str) else d.get("ipAddress")
+    node = d.get("nodeName")
+    sector = d.get("sectorName")
+    small = d.get("smallCellName")
+    serial = d.get("serialNumber")
+
+    if not fw:
+        die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
+
+    # 3) NetBox patch (idempotent)
+    cf_patch = {}
+    if cf.get("fw_version") != fw:
+        cf_patch["fw_version"] = fw
+        log(f"CF: fw_version -> {fw}")
+    if node and cf.get("nodeName") != node:
+        cf_patch["nodeName"] = node
+        log(f"CF: nodeName -> {node}")
+    if sector and cf.get("sectorName") != sector:
+        cf_patch["sectorName"] = sector
+        log(f"CF: sectorName -> {sector}")
+    if small and cf.get("smallCellName") != small:
+        cf_patch["smallCellName"] = small
+        log(f"CF: smallCellName -> {small}")
+
+    dev_patch = {}
+    if serial and (dev_full.get("serial") != serial):
+        dev_patch["serial"] = serial
+        log(f"DEV: serial -> {serial}")
+    if cf_patch:
+        dev_patch["custom_fields"] = cf_patch
+    nb.patch_device(dev_id, dev_patch)
+
+    # 4) IP handling (skip if placeholder)
+    ip_is_placeholder = (ip_from_cloud in (None, "", "0.0.0.0"))
+    ip_out_for_status = current_nb_ip  # default to current NB IP
+
+    if ip_is_placeholder:
+        log(f"IP: cloud reported placeholder '{ip_from_cloud}', skipping IP changes; keeping NetBox ip(s) as-is")
+    else:
+        iface_id = nb.ensure_eth0(dev_id)
+
+        ip_rec = nb.get_ip_by_addr(ip_from_cloud)
+        if ip_rec is None:
+            log(f"IP: create new {ip_from_cloud} on iface {iface_id}")
+            ip_id = nb.create_ip_for_iface(ip_from_cloud, iface_id)
+        else:
+            ip_id = ip_rec["id"]
+            assigned_type = ip_rec.get("assigned_object_type") or ""
+            assigned_id = ip_rec.get("assigned_object_id")
+            if not assigned_type:
+                if not nb.assign_ip_to_iface(ip_id, iface_id):
+                    die(4, f"FAIL assign IP {ip_from_cloud} to iface {iface_id}")
+            elif assigned_type == "dcim.interface":
+                if str(assigned_id) != str(iface_id):
+                    other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
+                    log(f"IP: moving {ip_from_cloud} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id}")
+                    # Clear old device primary if necessary
+                    if other_dev:
+                        r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
+                        if r.status_code == 200:
+                            old_primary_id = (r.json().get("primary_ip4") or {}).get("id")
+                            if str(old_primary_id) == str(ip_id):
+                                S_NB.patch(nb._url(f"/api/dcim/devices/{other_dev}/"),
+                                           headers=nb._h(), data=json.dumps({"primary_ip4": None}),
+                                           timeout=REQ_TIMEOUT)
+                    if not nb.assign_ip_to_iface(ip_id, iface_id):
+                        die(4, f"FAIL move IP {ip_from_cloud} to iface {iface_id}")
+            else:
+                die(4, f"FAIL IP {ip_from_cloud} assigned to {assigned_type}")
+
+        nb.device_set_primary_ip4(dev_id, ip_id)
+        ip_out_for_status = ip_from_cloud
+
+        # PRUNE all other IPs on this device (default behavior)
+        all_ips = nb.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get("id")
+            if str(rid) == str(ip_id):
+                continue
+            addr = rec.get("address")
+            log(f"IP: pruning stale {addr} (id={rid}) from device {dev_id}")
+            nb.delete_ip(rid)
+
+    print(f"OK {hostname} ip={ip_out_for_status or 'NONE'} fw={fw} node={node} sector={sector} small={small}")
+
+# ------------ CLI ------------
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname (only if online)")
+    ap.add_argument("hostname", help="Device name in NetBox")
+    ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
+    args = ap.parse_args()
+
+    CHATTY = bool(args.chatty)  # module-scope assignment
+    try:
+        run(args.hostname)
+    except requests.RequestException as e:
+        die(3, f"FAIL network error: {e}")

files/nb_onedevice_update.py

@@ -2,10 +2,13 @@
 """
 nb_sync_one_device.py
 
-Update a single NetBox device from Cloud by hostname — ONLY if device is online.
+Update a single NetBox device from Cloud by hostname.
 
 Behavior
-- Liveness gate via /v1/devices/{cloud_id}/liveness (no changes if offline).
+- Liveness gate via /v2/devices/{cloud_id}/liveness.
+  * If online: keep ORIGINAL behavior (Cloud is SoT, including IP).
+  * If offline: query Subsystem for the device and use Subsystem IP as fallback,
+    then continue with normal NetBox updates.
 - Updates NetBox fields from Cloud:
   custom_fields.fw_version ← firmwareVersion
   custom_fields.nodeName   ← nodeName
@@ -13,17 +16,17 @@ Behavior
   custom_fields.smallCellName ← smallCellName
   serial ← serialNumber
 - IP handling:
-  If Cloud ipAddress is valid (not None/""/"0.0.0.0"):
+  If chosen IP (Cloud when online, Subsystem when offline) is valid (not None/""/"0.0.0.0"):
     ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
     then PRUNE all other IPs on this device (default).
-  If Cloud ipAddress is placeholder/invalid:
+  If chosen IP is placeholder/invalid:
     skip IP changes and do not prune.
 - --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
-- NEW: Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+- Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
 
 Exit codes:
   0 = success
-  1 = not found / missing data / offline
+  1 = not found / missing data
   3 = network/HTTP error
   4 = NetBox update error
 """
@@ -31,7 +34,7 @@ Exit codes:
 import sys
 import json
 import argparse
-from typing import Optional, Union, List, Dict
+from typing import Optional, Union, List, Dict, Any
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -40,8 +43,15 @@ from urllib3.util.retry import Retry
 # ------------ HARD-CODED CONFIG (per Pavel) ------------
 NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
 NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
-CLOUD_API_BASE = "https://cloud.ikeja.co.za/v1/devices"
+
-CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzY2OTk3ODQ2LCJleHAiOjE3Njk1ODk4NDZ9.aBuEOQOC6i5jX6Ixwb3gabzV4uIeYJdbpJodxQy1DkE"
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v2/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
+
+
+# NEW: Subsystem fallback (only used when Cloud liveness isConnected=false)
+SUBSYSTEM_BASE = "https://subsystem.ikeja.co.za"
+SUBSYSTEM_TOKEN = "HJL+&XRCoeHwh5?13@gxvg86qD#kQfgc"
+SUBSYSTEM_OUTDOOR_ENDPOINT = "/customers/wave-devices/get-outdoor-devices"
 
 REQ_TIMEOUT = 30
 CHATTY = False
@@ -57,13 +67,14 @@ def _new_session() -> requests.Session:
         respect_retry_after_header=True,
     )
     adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
-    s.mount('http://', adapter)
+    s.mount("http://", adapter)
-    s.mount('https://', adapter)
+    s.mount("https://", adapter)
-    s.headers['Accept'] = 'application/json'
+    s.headers["Accept"] = "application/json"
     return s
 
 S_NB = _new_session()
 S_CL = _new_session()
+S_SUB = _new_session()
 
 # ------------ Logging / status helpers ------------
 def log(msg: str):
@@ -71,14 +82,21 @@ def log(msg: str):
         print(msg, file=sys.stderr)
 
 def die(code: int, msg: str):
-    # Single-line result: stdout on success, stderr on failure
     print(msg, file=sys.stdout if code == 0 else sys.stderr)
     raise SystemExit(code)
 
+def _is_placeholder_ip(v: Any) -> bool:
+    return v in (None, "", "0.0.0.0")
+
+def _clean_ip(v: Any) -> Any:
+    if isinstance(v, str):
+        return v.strip()
+    return v
+
 # ------------ NetBox API ------------
 class NetBox:
     def __init__(self, base: str, token: str):
-        self.base = base.rstrip('/')
+        self.base = base.rstrip("/")
         self.token = token
 
     def _h(self):
@@ -132,7 +150,6 @@ class NetBox:
         if r.status_code == 201:
             return r.json()["id"]
         if r.status_code == 400:
-            # race: read again
             ifid = self.get_iface_id(dev_id, "eth0")
             if ifid:
                 return ifid
@@ -162,10 +179,7 @@ class NetBox:
     def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
         log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
         r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
-                       data=json.dumps({
+                       data=json.dumps({"assigned_object_type": "dcim.interface", "assigned_object_id": iface_id}),
-                           "assigned_object_type": "dcim.interface",
-                           "assigned_object_id": iface_id
-                       }),
                        timeout=REQ_TIMEOUT)
         return 200 <= r.status_code < 300
 
@@ -182,24 +196,12 @@ class NetBox:
             return (r.json().get("device") or {}).get("id")
         return None
 
-    # --- Helpers for pruning ---
     def list_device_ips(self, dev_id: int) -> List[Dict]:
         r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
         if r.status_code == 200:
             return r.json().get("results") or []
-        ips: List[Dict] = []
+        return []
-        r2 = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
-                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
-        if r2.status_code == 200:
-            for iface in (r2.json().get("results") or []):
-                ifid = iface.get("id")
-                r3 = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
-                              params={"assigned_object_type": "dcim.interface",
-                                      "assigned_object_id": ifid, "limit": 1000}, timeout=REQ_TIMEOUT)
-                if r3.status_code == 200:
-                    ips.extend(r3.json().get("results") or [])
-        return ips
 
     def delete_ip(self, ip_id: int) -> None:
         log(f"NB: delete IP id={ip_id}")
@@ -232,6 +234,33 @@ class Cloud:
             die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
         return r.json()
 
+# ------------ Subsystem fallback ------------
+class Subsystem:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip("/")
+        self.token = token
+
+    def _h(self):
+        return {"token": self.token, "Accept": "application/json"}
+
+    def fetch_outdoor_devices(self) -> List[Dict[str, Any]]:
+        url = f"{self.base}{SUBSYSTEM_OUTDOOR_ENDPOINT}"
+        log("SUBSYS: fetch outdoor devices list")
+        r = S_SUB.post(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Subsystem POST get-outdoor-devices HTTP={r.status_code}")
+        data = r.json()
+        return data.get("device_list") or []
+
+    def find_by_name(self, name: str) -> Optional[Dict[str, Any]]:
+        devs = self.fetch_outdoor_devices()
+        want = name.lower()
+        for rec in devs:
+            n = rec.get("name")
+            if isinstance(n, str) and n.lower() == want:
+                return rec
+        return None
+
 # ------------ Core ------------
 def _primary_ip4_text(dev_json: dict) -> Optional[str]:
     p = dev_json.get("primary_ip4") or {}
@@ -243,6 +272,7 @@ def _primary_ip4_text(dev_json: dict) -> Optional[str]:
 def run(hostname: str) -> None:
     nb = NetBox(NB_URL, NB_TOKEN)
     cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+    sub = Subsystem(SUBSYSTEM_BASE, SUBSYSTEM_TOKEN)
 
     dev = nb.get_device_by_name(hostname)
     if not dev:
@@ -252,7 +282,6 @@ def run(hostname: str) -> None:
     dev_full = nb.get_device(dev_id)
     cf = dev_full.get("custom_fields") or {}
 
-    # NEW: log upgrade command if present
     upgrade_cmd = cf.get("upgrade_cmd")
     log(f"NB: upgrade_cmd={upgrade_cmd}")
 
@@ -260,17 +289,20 @@ def run(hostname: str) -> None:
     if cloud_id in (None, "", "null"):
         die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
 
-    # 1) Liveness gate
+    # 1) Cloud liveness gate (ONLINE path stays as-is)
     live = cl.device_liveness(cloud_id)
-    if not bool(live.get("isConnected")):
+    cloud_connected = bool(live.get("isConnected"))
-        die(1, f"FAIL {hostname} cloud_id={cloud_id} device is not online")
+    if cloud_connected:
+        log(f"CL: liveness isConnected=true for cloud_id={cloud_id}; using Cloud ipAddress as source of truth")
+    else:
+        log(f"CL: liveness isConnected=false for cloud_id={cloud_id}; falling back to Subsystem for IP")
 
     current_nb_ip = _primary_ip4_text(dev_full)
 
-    # 2) Cloud detail
+    # 2) Cloud detail (still used for fw/node/sector/small/serial)
     d = cl.device_detail(cloud_id)
     fw = d.get("firmwareVersion") or d.get("version")
-    ip_from_cloud = (d.get("ipAddress") or "").strip() if isinstance(d.get("ipAddress"), str) else d.get("ipAddress")
+    cloud_ip = _clean_ip(d.get("ipAddress"))
     node = d.get("nodeName")
     sector = d.get("sectorName")
     small = d.get("smallCellName")
@@ -279,6 +311,20 @@ def run(hostname: str) -> None:
     if not fw:
         die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
 
+    # Choose IP source
+    chosen_ip = cloud_ip
+    ip_source = "cloud"
+
+    if not cloud_connected:
+        sub_rec = sub.find_by_name(hostname)
+        if not sub_rec:
+            # No subsystem record -> keep previous behavior: fail because offline and no fallback source
+            die(1, f"FAIL {hostname} cloud_id={cloud_id} device offline and not found in Subsystem")
+        sub_ip = _clean_ip(sub_rec.get("ip"))
+        chosen_ip = sub_ip
+        ip_source = "subsystem"
+        log(f"SUBSYS: {hostname} ip={sub_ip} (fallback)")
+
     # 3) NetBox patch (idempotent)
     cf_patch = {}
     if cf.get("fw_version") != fw:
@@ -303,29 +349,30 @@ def run(hostname: str) -> None:
     nb.patch_device(dev_id, dev_patch)
 
     # 4) IP handling (skip if placeholder)
-    ip_is_placeholder = (ip_from_cloud in (None, "", "0.0.0.0"))
+    chosen_ip = _clean_ip(chosen_ip)
+    ip_is_placeholder = _is_placeholder_ip(chosen_ip)
     ip_out_for_status = current_nb_ip  # default to current NB IP
 
     if ip_is_placeholder:
-        log(f"IP: cloud reported placeholder '{ip_from_cloud}', skipping IP changes; keeping NetBox ip(s) as-is")
+        log(f"IP: {ip_source} reported placeholder '{chosen_ip}', skipping IP changes; keeping NetBox ip(s) as-is")
     else:
         iface_id = nb.ensure_eth0(dev_id)
 
-        ip_rec = nb.get_ip_by_addr(ip_from_cloud)
+        ip_rec = nb.get_ip_by_addr(chosen_ip)
         if ip_rec is None:
-            log(f"IP: create new {ip_from_cloud} on iface {iface_id}")
+            log(f"IP: create new {chosen_ip} on iface {iface_id} (source={ip_source})")
-            ip_id = nb.create_ip_for_iface(ip_from_cloud, iface_id)
+            ip_id = nb.create_ip_for_iface(chosen_ip, iface_id)
         else:
             ip_id = ip_rec["id"]
             assigned_type = ip_rec.get("assigned_object_type") or ""
             assigned_id = ip_rec.get("assigned_object_id")
             if not assigned_type:
                 if not nb.assign_ip_to_iface(ip_id, iface_id):
-                    die(4, f"FAIL assign IP {ip_from_cloud} to iface {iface_id}")
+                    die(4, f"FAIL assign IP {chosen_ip} to iface {iface_id}")
             elif assigned_type == "dcim.interface":
                 if str(assigned_id) != str(iface_id):
                     other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
-                    log(f"IP: moving {ip_from_cloud} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id}")
+                    log(f"IP: moving {chosen_ip} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id} (source={ip_source})")
                     # Clear old device primary if necessary
                     if other_dev:
                         r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
@@ -336,16 +383,15 @@ def run(hostname: str) -> None:
                                            headers=nb._h(), data=json.dumps({"primary_ip4": None}),
                                            timeout=REQ_TIMEOUT)
                     if not nb.assign_ip_to_iface(ip_id, iface_id):
-                        die(4, f"FAIL move IP {ip_from_cloud} to iface {iface_id}")
+                        die(4, f"FAIL move IP {chosen_ip} to iface {iface_id}")
             else:
-                die(4, f"FAIL IP {ip_from_cloud} assigned to {assigned_type}")
+                die(4, f"FAIL IP {chosen_ip} assigned to {assigned_type}")
 
         nb.device_set_primary_ip4(dev_id, ip_id)
-        ip_out_for_status = ip_from_cloud
+        ip_out_for_status = chosen_ip
 
         # PRUNE all other IPs on this device (default behavior)
-        all_ips = nb.list_device_ips(dev_id)
+        for rec in nb.list_device_ips(dev_id):
-        for rec in all_ips:
             rid = rec.get("id")
             if str(rid) == str(ip_id):
                 continue
@@ -357,12 +403,12 @@ def run(hostname: str) -> None:
 
 # ------------ CLI ------------
 if __name__ == "__main__":
-    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname (only if online)")
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname")
     ap.add_argument("hostname", help="Device name in NetBox")
     ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
     args = ap.parse_args()
 
-    CHATTY = bool(args.chatty)  # module-scope assignment
+    CHATTY = bool(args.chatty)
     try:
         run(args.hostname)
     except requests.RequestException as e:

files/netbox_subsystem_ikejanum_recursive.py

@@ -0,0 +1,885 @@
+#!/usr/bin/env python3
+"""
+Recursive single-target sync: Subsystem -> NetBox (Source of Truth), with depth.
+
+Usage:
+  ./netbox_subsystem_ikejanum <device_name> [--deep N]
+
+Behavior:
+- Start with <device_name>.
+- Sync from Subsystem into NetBox (create device if missing).
+- If device missing in Subsystem but present in NetBox -> set NetBox status=failed.
+- While syncing:
+    * If an IP is MOVED from another NetBox device -> enqueue that old device for sync (depth+1)
+    * If a stale IP is being DELETED from a device -> find Subsystem device that owns that IP and enqueue it (depth+1)
+
+Depth:
+- Default depth: 3
+- --deep N allowed, but N is hard-limited to <= 10.
+
+NEW:
+- --info-only: Print the raw Subsystem record (JSON) for the device and exit.
+  No NetBox changes, no recursion.
+
+Exit codes:
+  0 success
+  1 runtime error / not found (info-only)
+  2 config/arg error
+"""
+
+# =========================
+# USER CONFIG — EDIT HERE
+# =========================
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+
+SUBSYSTEM_BASE = "https://subsystem.ikeja.co.za"
+SUBSYSTEM_TOKEN = "HJL+&XRCoeHwh5?13@gxvg86qD#kQfgc"
+
+ROLE_CPE = 1
+TYPE_FOX100_CPE = 1
+TYPE_FOX200 = 2
+SITE_ID = 1
+
+DEFAULT_MIN_LAST_DETECTED_MINUTES = 0  # 0 = accept any age
+DEFAULT_DEEP = 3
+MAX_DEEP = 10
+# =========================
+
+import os
+import sys
+import json
+import time
+import logging
+import argparse
+from typing import Optional, Dict, Any, Tuple, List, Set, Deque
+from collections import deque
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+
+# ------------- Logging -------------
+def setup_logging(troubleshoot: bool) -> None:
+    level = logging.DEBUG if troubleshoot else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format='%(asctime)s %(levelname)s %(message)s',
+        datefmt='%H:%M:%S'
+    )
+
+def summarize_body(body: Any, limit: int = 500) -> str:
+    if body is None:
+        return "<none>"
+    if isinstance(body, (dict, list)):
+        s = json.dumps(body)
+    else:
+        s = str(body)
+    if len(s) > limit:
+        return s[:limit] + f"... (+{len(s)-limit}B)"
+    return s
+
+def is_ascii_or_die(label: str, value: str) -> None:
+    try:
+        value.encode('latin-1')
+    except UnicodeEncodeError:
+        bad = ''.join(ch for ch in value if ord(ch) > 127)
+        logging.error(
+            "%s contains non-ASCII characters (e.g. %r). Please paste the exact token without smart punctuation.",
+            label, bad,
+        )
+        raise SystemExit(2)
+
+# ------------- HTTP Session helpers -------------
+class Http:
+    def __init__(self, troubleshoot: bool = False):
+        self.s_nb = self._new_session()
+        self.s_subsystem = self._new_session()
+        self.troubleshoot = troubleshoot
+
+    def _new_session(self) -> requests.Session:
+        s = requests.Session()
+        retries = Retry(
+            total=3,
+            connect=3,
+            read=3,
+            status=3,
+            backoff_factor=0.75,
+            status_forcelist=(429, 500, 502, 503, 504),
+            allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+            respect_retry_after_header=True,
+        )
+        adapter = HTTPAdapter(max_retries=retries, pool_connections=16, pool_maxsize=32)
+        s.mount('http://', adapter)
+        s.mount('https://', adapter)
+        s.headers['Accept'] = 'application/json'
+        s.headers['Content-Type'] = 'application/json'
+        return s
+
+    def nb(self, method: str, url: str, token: str, **kw) -> requests.Response:
+        headers = kw.pop('headers', {})
+        headers['Authorization'] = f'Token {token}'
+        t0 = time.time()
+        resp = self.s_nb.request(method, url, headers=headers, timeout=30, **kw)
+        dt = (time.time() - t0) * 1000
+        if self.troubleshoot or resp.status_code >= 400:
+            logging.debug(
+                "NB %s %s [%s] %.1fms\n  req:%s\n  resp:%s",
+                method, url, resp.status_code, dt,
+                summarize_body(kw.get('data') or kw.get('json')),
+                summarize_body(resp.text),
+            )
+        return resp
+
+    def subsystem(self, method: str, url: str, token: str, **kw) -> requests.Response:
+        headers = kw.pop('headers', {})
+        headers['token'] = token
+        t0 = time.time()
+        resp = self.s_subsystem.request(method, url, headers=headers, timeout=30, **kw)
+        dt = (time.time() - t0) * 1000
+        if self.troubleshoot or resp.status_code >= 400:
+            logging.debug(
+                "SUBSYS %s %s [%s] %.1fms\n  resp:%s",
+                method, url, resp.status_code, dt,
+                summarize_body(resp.text),
+            )
+        return resp
+
+# ------------- Utility -------------
+def normalize_mac_from_subsystem(mac_raw: str) -> Optional[str]:
+    if not mac_raw:
+        return None
+    s = mac_raw.strip().replace(":", "").replace("-", "")
+    s = s.upper()
+    try:
+        chunks = [s[i:i+2] for i in range(0, len(s), 2)]
+        return ":".join(chunks)
+    except Exception:
+        logging.warning("  !! failed to normalize MAC from subsystem: %r", mac_raw)
+        return None
+
+def parse_last_detected(ts: Optional[str]) -> Optional[datetime]:
+    if not ts:
+        return None
+    try:
+        dt = parsedate_to_datetime(ts)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        else:
+            dt = dt.astimezone(timezone.utc)
+        return dt
+    except Exception:
+        logging.warning("  !! cannot parse last_detected: %r", ts)
+        return None
+
+def minutes_age(now_utc: datetime, past_utc: datetime) -> float:
+    return (now_utc - past_utc).total_seconds() / 60.0
+
+def ip_strip_prefix(ip_with_prefix: str) -> str:
+    # "10.0.0.1/32" -> "10.0.0.1"
+    return (ip_with_prefix or "").split("/")[0].strip()
+
+# ------------- Subsystem client (with caching) -------------
+class Subsystem:
+    def __init__(self, base: str, token: str, http: Http):
+        self.base = base.rstrip('/')
+        self.token = token
+        self.http = http
+        self._cache_devices: Optional[list[Dict[str, Any]]] = None
+
+    def fetch_outdoor_devices(self) -> list[Dict[str, Any]]:
+        url = f"{self.base}/customers/wave-devices/get-outdoor-devices"
+        r = self.http.subsystem("POST", url, self.token)
+        r.raise_for_status()
+        data = r.json()
+        return data.get("device_list") or []
+
+    def _ensure_cache(self) -> None:
+        if self._cache_devices is None:
+            self._cache_devices = self.fetch_outdoor_devices()
+
+    def find_device_by_name(self, target_name: str) -> Optional[Dict[str, Any]]:
+        self._ensure_cache()
+        assert self._cache_devices is not None
+        for rec in self._cache_devices:
+            name = (rec.get("name") or "").strip()
+            if name.lower() == target_name.lower():
+                return rec
+        return None
+
+    def find_device_by_ip(self, ip: str) -> Optional[Dict[str, Any]]:
+        if not ip:
+            return None
+        self._ensure_cache()
+        assert self._cache_devices is not None
+        ip = ip.strip()
+        for rec in self._cache_devices:
+            rip = (rec.get("ip") or "").strip()
+            if rip == ip:
+                return rec
+        return None
+
+# ------------- NetBox API wrappers -------------
+class NetBox:
+    def __init__(
+        self,
+        base: str,
+        token: str,
+        role_cpe: int,
+        type_fox100: int,
+        type_fox200: int,
+        site_id: int,
+        http: Http,
+        dry_run: bool = False,
+    ):
+        self.base = base.rstrip('/')
+        self.token = token
+        self.role_cpe = role_cpe
+        self.type_fox100 = type_fox100
+        self.type_fox200 = type_fox200
+        self.site_id = site_id
+        self.http = http
+        self.dry = dry_run
+        self.allow_moves = True
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def _req(self, method: str, path: str, **kw) -> requests.Response:
+        if self.dry and method in ("POST", "PATCH", "PUT", "DELETE"):
+            logging.info("DRY %s %s", method, path)
+            r = requests.Response()
+            if method == "POST":
+                r.status_code = 201
+                r._content = b'{"id": 0}'
+            else:
+                r.status_code = 200
+                r._content = b'{}'
+            r.headers['Content-Type'] = 'application/json'
+            return r
+        return self.http.nb(method, self._url(path), self.token, **kw)
+
+    # --- devices ---
+    def get_device_id_by_name(self, name: str) -> Optional[int]:
+        r = self._req("GET", "/api/dcim/devices/", params={"name": name})
+        r.raise_for_status()
+        data = r.json()
+        if data.get('results'):
+            return data['results'][0]['id']
+        return None
+
+    def get_device(self, dev_id: int) -> Dict[str, Any]:
+        r = self._req("GET", f"/api/dcim/devices/{dev_id}/")
+        r.raise_for_status()
+        return r.json()
+
+    def get_device_name(self, dev_id: int) -> Optional[str]:
+        r = self._req("GET", f"/api/dcim/devices/{dev_id}/")
+        if r.status_code == 200:
+            return r.json().get("name")
+        return None
+
+    def create_device(self, name: str, dtype_id: int) -> Optional[int]:
+        payload = {
+            "name": name,
+            "role": self.role_cpe,
+            "device_type": dtype_id,
+            "site": self.site_id,
+            "status": "active",
+        }
+        r = self._req("POST", "/api/dcim/devices/", json=payload)
+        if r.status_code == 201:
+            dev_id = r.json().get('id')
+            logging.info("  -> created NetBox device name=%s id=%s", name, dev_id)
+            return dev_id
+        logging.error("device create failed (%s) HTTP=%s body=%s", name, r.status_code, summarize_body(r.text))
+        return None
+
+    def patch_device(self, dev_id: int, patch: Dict[str, Any]) -> bool:
+        if not patch:
+            return True
+        r = self._req("PATCH", f"/api/dcim/devices/{dev_id}/", json=patch)
+        if 200 <= r.status_code < 300:
+            return True
+        logging.error("device patch failed dev=%s HTTP=%s body=%s", dev_id, r.status_code, summarize_body(r.text))
+        return False
+
+    def set_primary_ip4_if_changed(self, dev: Dict[str, Any], ip_id: int) -> bool:
+        curr = (dev.get('primary_ip4') or {}).get('id')
+        if curr == ip_id:
+            return True
+        r = self._req("PATCH", f"/api/dcim/devices/{dev['id']}/", json={"primary_ip4": ip_id})
+        ok = 200 <= r.status_code < 300
+        if not ok:
+            logging.error("set primary_ip4 failed dev=%s HTTP=%s body=%s", dev['id'], r.status_code, summarize_body(r.text))
+        return ok
+
+    def set_status(self, dev_id: int, status: str) -> bool:
+        r = self._req("PATCH", f"/api/dcim/devices/{dev_id}/", json={"status": status})
+        if 200 <= r.status_code < 300:
+            return True
+        logging.error("status patch failed dev=%s status=%s HTTP=%s body=%s", dev_id, status, r.status_code, summarize_body(r.text))
+        return False
+
+    def add_journal_entry(self, dev_id: int, comments: str, kind: str = "info") -> bool:
+        payload = {
+            "assigned_object_type": "dcim.device",
+            "assigned_object_id": dev_id,
+            "kind": kind,
+            "comments": comments,
+        }
+        r = self._req("POST", "/api/extras/journal-entries/", json=payload)
+        if 200 <= r.status_code < 300:
+            return True
+        logging.error(
+            "journal entry create failed dev=%s kind=%s HTTP=%s body=%s",
+            dev_id, kind, r.status_code, summarize_body(r.text)
+        )
+        return False
+
+    # --- interfaces ---
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = self._req("GET", "/api/dcim/interfaces/", params={"device_id": dev_id, "name": name})
+        r.raise_for_status()
+        data = r.json()
+        if data.get('results'):
+            return data['results'][0]['id']
+        return None
+
+    def ensure_eth0(self, dev_id: int) -> Optional[int]:
+        ifid = self.get_iface_id(dev_id, 'eth0')
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = self._req("POST", "/api/dcim/interfaces/", json=payload)
+        if r.status_code == 201:
+            return r.json()['id']
+        if r.status_code == 400:
+            return self.get_iface_id(dev_id, 'eth0')
+        logging.error("interface create failed device=%s HTTP=%s body=%s", dev_id, r.status_code, summarize_body(r.text))
+        return None
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = self._req("GET", f"/api/dcim/interfaces/{iface_id}/")
+        if r.status_code == 200:
+            return (r.json().get('device') or {}).get('id')
+        return None
+
+    def get_interface_detail(self, iface_id: int) -> Optional[Dict[str, Any]]:
+        r = self._req("GET", f"/api/dcim/interfaces/{iface_id}/")
+        if 200 <= r.status_code < 300:
+            return r.json()
+        logging.error("interface fetch failed iface=%s HTTP=%s body=%s", iface_id, r.status_code, summarize_body(r.text))
+        return None
+
+    # --- IP inventory helpers ---
+    def list_device_ips(self, dev_id: int) -> list[Dict[str, Any]]:
+        r = self._req("GET", "/api/ipam/ip-addresses/", params={"device_id": dev_id, "limit": 1000})
+        if r.status_code == 200:
+            return r.json().get('results') or []
+        # fallback by interface
+        ips: list[Dict[str, Any]] = []
+        r2 = self._req("GET", "/api/dcim/interfaces/", params={"device_id": dev_id, "limit": 1000})
+        if r2.status_code == 200:
+            for iface in (r2.json().get('results') or []):
+                ifid = iface.get('id')
+                r3 = self._req("GET", "/api/ipam/ip-addresses/", params={
+                    "assigned_object_type": "dcim.interface",
+                    "assigned_object_id": ifid,
+                    "limit": 1000,
+                })
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get('results') or [])
+        return ips
+
+    def prune_other_ips_with_details(self, dev_id: int, keep_ip_id: int) -> List[Tuple[int, str]]:
+        """
+        Delete all other IPs assigned to this device, keeping only keep_ip_id.
+        Returns list of (ip_id, address) that were removed.
+        """
+        removed: List[Tuple[int, str]] = []
+        all_ips = self.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get('id')
+            if rid == keep_ip_id:
+                continue
+            addr = rec.get('address')
+            logging.warning("  -> removing stale IP %s (id=%s) from device %s", addr, rid, dev_id)
+            self._req("DELETE", f"/api/ipam/ip-addresses/{rid}/")
+            if rid is not None and addr:
+                removed.append((int(rid), str(addr)))
+        return removed
+
+    # --- MAC helpers ---
+    def ensure_single_mac_on_iface(self, iface_id: int, mac_norm: str) -> Tuple[int, int]:
+        created = 0
+        deleted = 0
+        iface = self.get_interface_detail(iface_id)
+        if not iface:
+            logging.error("  !! cannot fetch interface detail for MAC sync (iface=%s)", iface_id)
+            return (0, 0)
+
+        macs = iface.get("mac_addresses") or []
+        matching_ids: list[int] = []
+        bad_ids: list[int] = []
+
+        for m in macs:
+            mid = m.get("id")
+            mval = (m.get("mac_address") or "").upper()
+            if mval == mac_norm:
+                if mid is not None:
+                    matching_ids.append(mid)
+            else:
+                if mid is not None:
+                    bad_ids.append(mid)
+
+        if not matching_ids:
+            payload = {
+                "mac_address": mac_norm,
+                "assigned_object_type": "dcim.interface",
+                "assigned_object_id": iface_id,
+            }
+            logging.info("  -> create MAC %s on iface %s", mac_norm, iface_id)
+            r = self._req("POST", "/api/dcim/mac-addresses/", json=payload)
+            if r.status_code == 201:
+                created += 1
+            else:
+                logging.error("  !! MAC create failed iface=%s mac=%s HTTP=%s body=%s", iface_id, mac_norm, r.status_code, summarize_body(r.text))
+        else:
+            extra_ids = matching_ids[1:]
+            bad_ids.extend([i for i in extra_ids if i is not None])
+
+        for mid in bad_ids:
+            logging.warning("  -> removing stale/duplicate MAC entry id=%s from iface=%s", mid, iface_id)
+            self._req("DELETE", f"/api/dcim/mac-addresses/{mid}/")
+            deleted += 1
+
+        return (created, deleted)
+
+    # --- IP addresses ---
+    def get_ip_by_address(self, addr: str) -> Dict[str, Any] | None:
+        r = self._req("GET", "/api/ipam/ip-addresses/", params={"address": f"{addr}/32"})
+        r.raise_for_status()
+        res = r.json().get('results') or []
+        return res[0] if res else None
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        r = self._req("PATCH", f"/api/ipam/ip-addresses/{ip_id}/", json={
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        })
+        return r.status_code == 200
+
+    def ensure_ip_for_device(self, dev_id: int, iface_id: int, addr: str) -> Tuple[Optional[int], str, Optional[int]]:
+        """
+        Ensure addr/32 exists and is assigned to iface_id on dev_id.
+        Returns (ip_id, action, old_device_id_if_moved)
+          action: created, reused, assigned, moved, skipped, error
+        """
+        ip_rec = self.get_ip_by_address(addr)
+        if not ip_rec:
+            payload = {
+                "address": f"{addr}/32",
+                "status": "active",
+                "assigned_object_type": "dcim.interface",
+                "assigned_object_id": iface_id,
+            }
+            r = self._req("POST", "/api/ipam/ip-addresses/", json=payload)
+            if r.status_code == 201:
+                return (r.json()['id'], "created", None)
+            logging.error("ip create failed iface=%s addr=%s HTTP=%s body=%s", iface_id, addr, r.status_code, summarize_body(r.text))
+            return (None, "error", None)
+
+        ip_id = ip_rec['id']
+        aot = ip_rec.get('assigned_object_type') or ''
+        if not aot:
+            ok = self.assign_ip_to_iface(ip_id, iface_id)
+            return (ip_id if ok else None, "assigned" if ok else "error", None)
+
+        if aot == 'dcim.interface':
+            assigned_ifid = ip_rec.get('assigned_object_id')
+            if str(assigned_ifid) == str(iface_id):
+                return (ip_id, "reused", None)
+
+            assigned_dev = self.get_device_id_of_interface(assigned_ifid) if assigned_ifid else None
+            if assigned_dev and str(assigned_dev) == str(dev_id):
+                logging.info("  -> IP %s already on same device (iface=%s); reusing.", addr, assigned_ifid)
+                return (ip_id, "reused", None)
+
+            if self.allow_moves:
+                old_dev_id = assigned_dev
+
+                # clear primary_ip4 on old device if needed
+                if old_dev_id:
+                    r_old = self._req("GET", f"/api/dcim/devices/{old_dev_id}/")
+                    if r_old.status_code == 200:
+                        old_primary_id = (r_old.json().get("primary_ip4") or {}).get("id")
+                        if str(old_primary_id) == str(ip_id):
+                            r_clr = self._req("PATCH", f"/api/dcim/devices/{old_dev_id}/", json={"primary_ip4": None})
+                            if not (200 <= r_clr.status_code < 300):
+                                logging.error("  -> cannot clear old device %s primary_ip4 for IP %s; aborting move", old_dev_id, addr)
+                                return (None, "error", None)
+
+                logging.warning(
+                    "  -> IP %s currently belongs to device %s (iface %s); moving to this device (iface %s)",
+                    addr, old_dev_id, assigned_ifid, iface_id,
+                )
+                r2 = self._req("PATCH", f"/api/ipam/ip-addresses/{ip_id}/", json={
+                    "assigned_object_type": "dcim.interface",
+                    "assigned_object_id": iface_id,
+                })
+                if r2.status_code == 200:
+                    return (ip_id, "moved", old_dev_id)
+                logging.error("  -> move failed for IP %s HTTP=%s body=%s", addr, r2.status_code, summarize_body(r2.text))
+                return (None, "error", None)
+
+            return (None, "skipped", None)
+
+        logging.warning("  -> IP %s assigned to %s; skipping.", addr, aot)
+        return (None, "skipped", None)
+
+# ------------- Core sync functions -------------
+def sync_from_subsystem_record(
+    target: str,
+    rec: Dict[str, Any],
+    nb: NetBox,
+    subsystem: Subsystem,
+    now_utc: datetime,
+    min_last_detected_minutes: int,
+    enqueue_fn,
+) -> Dict[str, Any]:
+    """
+    enqueue_fn(name: str, reason: str) -> None
+    """
+    summary = {
+        "target": target,
+        "subsystem_found": True,
+        "netbox_created": False,
+        "status_set_active": False,
+        "node_updated": False,
+        "mac_created": 0,
+        "mac_deleted": 0,
+        "ip_action": "none",
+        "ips_pruned": 0,
+        "primary_set": False,
+        "skipped_stale": False,
+        "errors": 0,
+    }
+
+    name = rec.get("name") or target
+    ip = rec.get("ip")
+    mac_raw = rec.get("mac")
+    connected_node = rec.get("connected_node")
+    last_detected = rec.get("last_detected")
+
+    dt_last = parse_last_detected(last_detected)
+    if min_last_detected_minutes > 0:
+        if dt_last is None:
+            logging.info("  -> skip: last_detected unavailable; requires <= %s minutes", min_last_detected_minutes)
+            summary["skipped_stale"] = True
+            return summary
+        age_min = minutes_age(now_utc, dt_last)
+        if age_min > min_last_detected_minutes:
+            logging.info("  -> skip: last_detected age %.1f min > allowed %s min", age_min, min_last_detected_minutes)
+            summary["skipped_stale"] = True
+            return summary
+
+    dev_id = nb.get_device_id_by_name(name)
+    if not dev_id:
+        logging.info("  -> device not found in NetBox; creating")
+        dev_id = nb.create_device(name, nb.type_fox100)
+        if not dev_id:
+            summary["errors"] += 1
+            return summary
+        summary["netbox_created"] = True
+
+    dev = nb.get_device(dev_id)
+    curr_status = dev.get("status", {}).get("value") if isinstance(dev.get("status"), dict) else dev.get("status")
+    if curr_status != "active":
+        active_reason = "device present in subsystem"
+        if min_last_detected_minutes > 0:
+            active_reason += f" and last_detected within {min_last_detected_minutes} minutes"
+        else:
+            active_reason += " and freshness filter accepts it"
+        if nb.set_status(dev_id, "active"):
+            nb.add_journal_entry(dev_id, f"setting to active because {active_reason}", kind="info")
+            summary["status_set_active"] = True
+            dev = nb.get_device(dev_id)
+        else:
+            summary["errors"] += 1
+
+    cf = dev.get("custom_fields") or {}
+    current_node = (cf.get("nodeName") if isinstance(cf, dict) else None)
+
+    patch: Dict[str, Any] = {}
+    if connected_node and connected_node != current_node:
+        patch.setdefault("custom_fields", {})["nodeName"] = connected_node
+        logging.info("  -> patch custom_fields.nodeName: %r -> %r", current_node, connected_node)
+        if nb.patch_device(dev_id, patch):
+            summary["node_updated"] = True
+            dev = nb.get_device(dev_id)
+
+    iface_id = nb.ensure_eth0(dev_id)
+    if not iface_id:
+        logging.error("  !! interface ensure/create failed for device=%s", dev_id)
+        summary["errors"] += 1
+        return summary
+
+    mac_norm = normalize_mac_from_subsystem(mac_raw) if mac_raw else None
+    if mac_norm:
+        logging.info("  -> ensure single MAC %s on iface %s", mac_norm, iface_id)
+        c, d = nb.ensure_single_mac_on_iface(iface_id, mac_norm)
+        summary["mac_created"] += c
+        summary["mac_deleted"] += d
+    else:
+        logging.info("  -> MAC missing/unusable from subsystem; skipping MAC sync")
+
+    if ip and str(ip).lower() != "null" and ip != "0.0.0.0":
+        logging.info("  -> ensure IP %s for device %s (iface %s)", ip, dev_id, iface_id)
+        ip_id, ip_action, old_dev_id = nb.ensure_ip_for_device(dev_id, iface_id, ip)
+        summary["ip_action"] = ip_action
+
+        # If we moved from some other NetBox device, enqueue it for repair
+        if ip_action == "moved" and old_dev_id:
+            old_name = nb.get_device_name(old_dev_id)
+            if old_name and old_name.lower() != name.lower():
+                enqueue_fn(old_name, f"ip_moved_away:{ip}")
+
+        if ip_id:
+            removed = nb.prune_other_ips_with_details(dev_id, ip_id)
+            summary["ips_pruned"] = len(removed)
+
+            # For each deleted stale IP, see who owns that IP in Subsystem and enqueue them.
+            for _rid, addr_pref in removed:
+                stale_ip = ip_strip_prefix(addr_pref)
+                rec2 = subsystem.find_device_by_ip(stale_ip)
+                if rec2:
+                    n2 = rec2.get("name")
+                    if n2 and n2.lower() != name.lower():
+                        enqueue_fn(n2, f"stale_ip_deleted:{stale_ip}")
+
+            dev = nb.get_device(dev_id)
+            before = (dev.get("primary_ip4") or {}).get("id")
+            if nb.set_primary_ip4_if_changed(dev, ip_id):
+                if str(before) != str(ip_id):
+                    summary["primary_set"] = True
+        else:
+            summary["errors"] += 1
+    else:
+        logging.info("  -> IP missing or 0.0.0.0; skipping IP sync")
+
+    return summary
+
+def mark_failed_if_in_netbox_only(target: str, nb: NetBox) -> Dict[str, Any]:
+    summary = {
+        "target": target,
+        "subsystem_found": False,
+        "netbox_exists": False,
+        "status_set_failed": False,
+        "errors": 0,
+    }
+    dev_id = nb.get_device_id_by_name(target)
+    if not dev_id:
+        return summary
+    summary["netbox_exists"] = True
+
+    dev = nb.get_device(dev_id)
+    curr_status = dev.get("status", {}).get("value") if isinstance(dev.get("status"), dict) else dev.get("status")
+    if curr_status == "failed":
+        return summary
+
+    failed_reason = "device missing in subsystem"
+    if nb.set_status(dev_id, "failed"):
+        nb.add_journal_entry(dev_id, f"setting to failed, because {failed_reason}", kind="warning")
+        summary["status_set_failed"] = True
+    else:
+        summary["errors"] += 1
+    return summary
+
+def one_liner_single(result: Dict[str, Any]) -> str:
+    t = result.get("target", "?")
+    if result.get("subsystem_found") is False:
+        if result.get("netbox_exists"):
+            if result.get("status_set_failed"):
+                return f"{t}: not in subsystem -> NetBox status set to failed"
+            return f"{t}: not in subsystem -> NetBox status NOT changed (error)"
+        return f"{t}: not in subsystem and not in NetBox -> nothing to do"
+
+    if result.get("skipped_stale"):
+        return f"{t}: subsystem record skipped due to last_detected freshness filter"
+
+    parts = []
+    if result.get("netbox_created"):
+        parts.append("created in NetBox")
+    if result.get("status_set_active"):
+        parts.append("status set to active")
+    if result.get("node_updated"):
+        parts.append("nodeName updated")
+    if (result.get("mac_created", 0) or 0) > 0 or (result.get("mac_deleted", 0) or 0) > 0:
+        parts.append(f"mac c{result.get('mac_created',0)}/d{result.get('mac_deleted',0)}")
+    ia = result.get("ip_action")
+    if ia and ia != "none":
+        parts.append(f"ip {ia}")
+    if (result.get("ips_pruned", 0) or 0) > 0:
+        parts.append(f"ips pruned={result.get('ips_pruned')}")
+    if result.get("primary_set"):
+        parts.append("primary_ip4 set")
+    if (result.get("errors") or 0) > 0:
+        parts.append(f"errors={result.get('errors')}")
+
+    if not parts:
+        return f"{t}: already in sync (no changes)"
+    return f"{t}: " + ", ".join(parts)
+
+# ------------- Depth driver -------------
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("device_name", help="Exact device name to sync (e.g. ikeja12345)")
+    ap.add_argument(
+        "--info-only",
+        action="store_true",
+        default=False,
+        help="Print Subsystem record for the device and exit (no NetBox changes)",
+    )
+    ap.add_argument(
+        "--min-last-detected-minutes",
+        type=int,
+        default=DEFAULT_MIN_LAST_DETECTED_MINUTES,
+        help="Require subsystem last_detected within N minutes (0 = accept any age)",
+    )
+    ap.add_argument(
+        "--deep",
+        type=int,
+        default=DEFAULT_DEEP,
+        help=f"Recursive depth (default {DEFAULT_DEEP}, max {MAX_DEEP})",
+    )
+    ap.add_argument("--troubleshoot", action='store_true', default=bool(os.getenv('TROUBLESHOOT')))
+    ap.add_argument("--dry-run", action='store_true', default=False)
+    args = ap.parse_args()
+
+    setup_logging(args.troubleshoot)
+
+    if not NB_TOKEN:
+        logging.error("Set NB_TOKEN in script")
+        return 2
+    if not SUBSYSTEM_TOKEN:
+        logging.error("Set SUBSYSTEM_TOKEN in script")
+        return 2
+
+    is_ascii_or_die("NB_TOKEN", NB_TOKEN)
+    is_ascii_or_die("SUBSYSTEM_TOKEN", SUBSYSTEM_TOKEN)
+
+    min_min = max(0, int(args.min_last_detected_minutes))
+    deep = int(args.deep)
+    if deep < 1:
+        deep = 1
+    if deep > MAX_DEEP:
+        logging.warning("[WARN] --deep %s requested, capping to %s", deep, MAX_DEEP)
+        deep = MAX_DEEP
+
+    http = Http(troubleshoot=args.troubleshoot)
+    nb = NetBox(
+        NB_URL,
+        NB_TOKEN,
+        ROLE_CPE,
+        TYPE_FOX100_CPE,
+        TYPE_FOX200,
+        SITE_ID,
+        http,
+        dry_run=args.dry_run,
+    )
+    subsystem = Subsystem(SUBSYSTEM_BASE, SUBSYSTEM_TOKEN, http)
+
+    # --info-only mode (no NetBox changes, no recursion)
+    if args.info_only:
+        rec = subsystem.find_device_by_name(args.device_name.strip())
+        if not rec:
+            print(f"NOT_FOUND {args.device_name.strip()} in Subsystem")
+            return 1
+        print(json.dumps(rec, indent=2, sort_keys=True))
+        return 0
+
+    now_utc = datetime.now(timezone.utc)
+
+    # BFS queue: (name, depth, reason)
+    q: Deque[Tuple[str, int, str]] = deque()
+    seen: Set[str] = set()
+
+    root = args.device_name.strip()
+    q.append((root, 0, "root"))
+    seen.add(root.lower())
+
+    totals = {
+        "processed": 0,
+        "subsystem_found": 0,
+        "netbox_failed_set": 0,
+        "created": 0,
+        "errors": 0,
+        "enqueued": 0,
+    }
+
+    def enqueue(name: str, reason: str, parent_depth: int = 0) -> None:
+        nonlocal q, seen, totals, deep
+        if not name:
+            return
+        k = name.lower()
+        if k in seen:
+            return
+        next_depth = parent_depth + 1
+        if next_depth > deep:
+            return
+        seen.add(k)
+        q.append((name, next_depth, reason))
+        totals["enqueued"] += 1
+        logging.info("  -> enqueue depth=%s name=%s reason=%s", next_depth, name, reason)
+
+    per_device_one_liners: List[str] = []
+
+    try:
+        while q:
+            name, d, reason = q.popleft()
+            totals["processed"] += 1
+            logging.info("[INFO] depth=%s name=%s reason=%s", d, name, reason)
+
+            rec = subsystem.find_device_by_name(name)
+            if rec:
+                totals["subsystem_found"] += 1
+
+                def enq_child(child_name: str, child_reason: str) -> None:
+                    enqueue(child_name, child_reason, parent_depth=d)
+
+                res = sync_from_subsystem_record(
+                    name, rec, nb, subsystem, now_utc, min_min, enq_child
+                )
+                if res.get("netbox_created"):
+                    totals["created"] += 1
+                if (res.get("errors") or 0) > 0:
+                    totals["errors"] += 1
+                per_device_one_liners.append(one_liner_single(res))
+            else:
+                res2 = mark_failed_if_in_netbox_only(name, nb)
+                if res2.get("status_set_failed"):
+                    totals["netbox_failed_set"] += 1
+                if (res2.get("errors") or 0) > 0:
+                    totals["errors"] += 1
+                per_device_one_liners.append(one_liner_single(res2))
+
+    except Exception as e:
+        logging.exception("Unhandled exception in deep sync: %s", e)
+        print(f"{root}: error (exception during deep sync)")
+        return 1
+
+    for line in per_device_one_liners:
+        print(line)
+
+    print(
+        f"SUMMARY: root={root} processed={totals['processed']} "
+        f"subsys_found={totals['subsystem_found']} created={totals['created']} "
+        f"failed_set={totals['netbox_failed_set']} enqueued={totals['enqueued']} errors={totals['errors']}"
+    )
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

files/rabbit-client.sh

@@ -22,6 +22,9 @@ ROUTING_KEY="${ROUTING_KEY:-}"    # irrelevant when EXCHANGE is empty
 # Polling interval when no messages
 SLEEP_SECS="${SLEEP_SECS:-1}"
 
+# GO/NO-GO gate bypass (requested): set ignore_gonogo=true to ignore gate
+ignore_gonogo="${ignore_gonogo:-false}"
+
 # Paths
 APP_ROOT="/opt/containers/ansible-worker/app"
 NBPLAY="${APP_ROOT}/bin/nbplay"
@@ -46,6 +49,39 @@ log()  { printf '[consumer] %s\n' "$*"; }
 warn() { printf '[consumer][WARN] %s\n' "$*" >&2; }
 err()  { printf '[consumer][ERROR] %s\n' "$*" >&2; }
 
+# --- GO/NO-GO gate helpers (surgical add) ---
+gate_sleep_secs() {
+  # Random 2..5 seconds inclusive
+  echo $(( (RANDOM % 4) + 2 ))
+}
+
+gate_is_go() {
+  # Bypass if ignore_gonogo is true-ish
+  case "${ignore_gonogo,,}" in
+    true|1|yes|y) return 0 ;;
+  esac
+
+  # Fail-closed: any error/unreachable => NO-GO
+  local body val
+  if ! body="$(curl -fsS --connect-timeout 2 --max-time 3 "http://10.210.12.2:8090/data/go_nogo.txt" 2>/dev/null)"; then
+    return 1
+  fi
+
+  # Must match ^go$ (allow trailing newline in file)
+  val="$(printf '%s' "$body" | tr -d '\r' | head -n1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+  val="${val,,}"
+  [[ "$val" == "go" ]]
+}
+
+gate_block_if_needed() {
+  # If gate is closed, sleep random 2..5 seconds and signal caller to continue loop
+  if gate_is_go; then
+    return 0
+  fi
+  sleep "$(gate_sleep_secs)"
+  return 1
+}
+
 # Safely append string options into an array using eval (so quotes are honored).
 append_opts() {
   local opts_str="$1"
@@ -64,7 +100,6 @@ dispatch_task() {
   task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
   task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
 
-
   # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
   # Supports both historic 'afterupgrade_check' and current 'afterupgrade_indoor_check'
   if [[ "$task" == "afterupgrade_indoor_check" || "$task" == "afterupgrade_check" ]]; then
@@ -78,6 +113,7 @@ dispatch_task() {
     target_ver="$(jq -r '.target_version // ""' <<<"$json")"
     tv_full="$(jq -r '.target_version_full // ""' <<<"$json")"
     schema="$(jq -r '.schema_version // ""' <<<"$json")"
+    is_run_by="$(jq -r '.is_run_by // ""' <<<"$json")"
 
     # Backfill target_version from target_version_full if missing (e.g., fox200-2.2.1-r6801.bin → 2.2.1-r6801)
     if [[ -z "$target_ver" && -n "$tv_full" ]]; then
@@ -96,9 +132,9 @@ dispatch_task() {
     task_options+=" -e target_version='$(esc "$target_ver")'"
     task_options+=" -e target_version_full='$(esc "$tv_full")'"
     task_options+=" -e schema_version='$(esc "$schema")'"
+    task_options+=" -e is_run_by='$(esc "$is_run_by")'"
   fi
 
-
   if [[ -z "$device" || -z "$task" ]]; then
     warn "payload missing required keys (inscope_device/task_name). Skipping."
     return 0
@@ -136,6 +172,12 @@ dispatch_task() {
     rnd=$(( (RANDOM % 4) + 1 ))   # 1..4
     total_h=$(( ceil_h + rnd - 1 ))
 
+    # NEW: if rebootin is huge (>=18h), convert to immediate reboot (0h)
+    if (( total_h >= 19 )); then
+      total_h=0
+      log "Adjusted _tonight rebootin to 0h because computed value was >=18h"
+    fi
+
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
   elif [[ "$base_task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
@@ -146,6 +188,14 @@ dispatch_task() {
     n="${BASH_REMATCH[1]}"
     base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
     extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$base_task" =~ ^update-reboot-scheduler_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-reboot-scheduler"
+    extra_nbplay_opts+=("-erebootin=${n}")
+  elif [[ "$base_task" =~ ^(update-reboot[0-9]{3}-scheduler)_([0-9]{1,4})$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+    n="${BASH_REMATCH[2]}"
+    extra_nbplay_opts+=("-erebootin=${n}")
   elif [[ "$base_task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
     # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin=<n>
     n="${BASH_REMATCH[1]}"
@@ -160,7 +210,7 @@ dispatch_task() {
   fi
 
   # --- Default rebootin for reboot-family when not explicitly provided ---
-  if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot)$ ]]; then
+  if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot|update-reboot-scheduler|update-reboot[0-9]{3}-scheduler)$ ]]; then
     # only set if neither task_options nor extra_nbplay_opts already contain rebootin
     if [[ "$task_options" != *"rebootin="* ]] && ! printf '%s\n' "${extra_nbplay_opts[@]}" | grep -q 'rebootin='; then
       # restore legacy behavior: immediate reboot if none specified
@@ -221,6 +271,11 @@ if [[ -n "$EXCHANGE" ]]; then
   # Single-queue consume loop (unchanged branch)
   log "Press Ctrl+C to stop."
   while :; do
+    # GO/NO-GO gate: do NOT dequeue unless gate is GO
+    if ! gate_block_if_needed; then
+      continue
+    fi
+
     RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{
       "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000
     }')"
@@ -270,6 +325,11 @@ else
 
   # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over.
   while :; do
+    # GO/NO-GO gate: do NOT dequeue unless gate is GO
+    if ! gate_block_if_needed; then
+      continue
+    fi
+
     local_got_message=0
 
     for Q in "${QUEUE_LIST[@]}"; do