0827

files/ansible-playbooks/afterupgrade_check.yml

@@ -23,6 +23,9 @@
     # Do NOT self-reference max_attempts. We’ll normalize below.
     max_attempts_default: 3
 
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
   tasks:
     # ---- Normalize metadata safely (no self-referential defaults) ----
     - name: Normalize metadata (no clever transforms)
@@ -35,6 +38,13 @@
         # preserve the original string verbatim for all subsequent retries
         target_version_full: "{{ target_version | default('') }}"
 
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
     - name: Show received metadata
       ansible.builtin.debug:
         msg:
@@ -116,7 +126,65 @@
     - name: Evaluate version match (full-string contains check)
       when: nc_probe.rc == 0 and banner_probe.rc == 0
       ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
 
     # ---- Journaling paths ----
     # Success: banner matches expected full target_version
@@ -129,8 +197,9 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
 
     - name: Publish success journal to control queue
       when: journal_success_payload is defined
@@ -195,7 +264,7 @@
           task_name: "journal_add"
           task_result: >-
             afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
             Correlation={{ correlation_id }} Original={{ original_emitted_at }}
 
     - name: Publish mismatch journal to control queue

files/ansible-playbooks/afterupgrade_check_original.yml

@@ -0,0 +1,363 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_withcurl.yml

@@ -0,0 +1,437 @@
+---
+- name: After-upgrade verification (banner check + reporting)
+  hosts: all
+  gather_facts: no
+
+  # RabbitMQ + defaults (match the big script)
+  vars:
+    rmq_host: "10.210.12.2"
+    rmq_port: 15672
+    rmq_user: "admin"
+    rmq_pass: "change_me"
+    rmq_vhost: "app"
+    rmq_exchange: "controls"
+    control_queue: "queue_controls"
+
+# --- Manual run defaults (so we can execute without -e) --- 
+# These are safe to leave here; anything passed via -e will still override them.
+    attempt: 1 
+    max_attempts: 3 
+    current_delay_sec: 300 
+    correlation_id: "6f680073dc7c" 
+    original_emitted_at: "2025-10-30T18:46:52Z" 
+    target_version: "2.2.3 rev 9800" 
+    # Intentionally keep this empty to exercise the target_version_effective logic. 
+    target_version_full: "" 
+    schema_version: 1
+
+
+    # Probing/SSH defaults
+    tcp_port: 22
+    nc_timeout: 5
+    ssh_user: "{{ ansible_user | default('root') }}"
+    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
+    ssh_timeout: 10
+
+    # Do NOT self-reference max_attempts. We’ll normalize below.
+    max_attempts_default: 3
+
+    # --- Hardcoded cloud API bearer (per request) ---
+    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzU5NzMxMzk1LCJleHAiOjE3NjIzMjMzOTV9.C7XV-QHIsLPZTxavv1eU361p0KTpiEPfDv3AUTmAqG8"
+
+  tasks:
+    # ---- Normalize metadata safely (no self-referential defaults) ----
+    - name: Normalize metadata (no clever transforms)
+      ansible.builtin.set_fact:
+        attempt: "{{ (attempt | default(1)) | int }}"
+        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
+        correlation_id: "{{ correlation_id | default('') }}"
+        original_emitted_at: "{{ original_emitted_at | default('') }}"
+        target_version: "{{ target_version | default('') }}"
+        # preserve the original string verbatim for all subsequent retries
+        target_version_full: "{{ target_version | default('') }}"
+
+    - name: Derive effective target version (avoid extra-var masking)
+      ansible.builtin.set_fact:
+        target_version_effective: >-
+          {{ (target_version_full | default('') | trim)
+              if (target_version_full | default('') | trim)
+              else (target_version | default('') | trim) }}
+
+    - name: Show received metadata
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version(full)={{ target_version_full }}"
+
+    # ---- Fast TCP reachability probe (controller-side) ----
+    - name: Check if TCP/{{ tcp_port }} is reachable with nc
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Build failure journal (no TCP connectivity) + mark retry
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (no TCP connectivity)
+      when: nc_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_tcp_fail
+      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
+
+    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
+    - name: Stop host after TCP failure (we’ll schedule or close out below)
+      when: nc_probe.rc != 0
+      ansible.builtin.meta: noop
+
+    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
+    - name: Probe banner via SSH from controller (classic extraction)
+      when: nc_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
+      register: banner_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Show the current version (banner line)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.debug:
+        msg: "{{ banner_probe.stdout | trim }}"
+
+    - name: Evaluate version match (full-string contains check)
+      when: nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
+
+    # ---- Read eth0 MAC (only after confirmed version match) ----
+    - name: Read eth0 MAC address via SSH
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ ssh_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ ssh_pass }}' \
+        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
+          "${USER}@${HOST}" \
+          "cat /sys/class/net/eth0/address | tr -d '\n'"
+      register: mac_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Set eth0_macaddress fact
+      when: mac_probe is defined and mac_probe.rc == 0
+      ansible.builtin.set_fact:
+        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
+
+    - name: Debug eth0_macaddress
+      when: eth0_macaddress is defined
+      ansible.builtin.debug:
+        msg: "eth0_macaddress={{ eth0_macaddress }}"
+
+    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
+    - name: Build URL-encoded MAC for cloud API
+      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
+
+    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        curl -sS -L --request PATCH --post301 --post302 \
+          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
+          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
+          --header "Content-Type: application/json" \
+          --header "Accept: application/json" \
+          --fail-with-body \
+          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
+      register: cloud_patch
+      changed_when: false
+      ignore_errors: true
+
+    - name: Flag cloud change result
+      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
+      ansible.builtin.set_fact:
+        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
+
+    - name: Debug cloud change result
+      when: cloud_change_ok is defined
+      ansible.builtin.debug:
+        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
+
+    # ---- Journaling paths ----
+    # Success: banner matches expected full target_version
+    - name: Build success journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_success_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
+
+    - name: Publish success journal to control queue
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_success_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_success
+      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
+
+    # NEW: send a control tag to clean up device state on success
+    - name: Build cleanup control payload (update_cleanup_success)
+      when: journal_success_payload is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        control_cleanup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "update_cleanup_success"
+
+    - name: Publish cleanup control message to control queue
+      when: control_cleanup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ control_cleanup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cleanup_success
+      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
+
+    # Mismatch: reachable & banner read, but not equal to target_version
+    - name: Build mismatch journal payload
+      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_mismatch_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
+
+    - name: Publish mismatch journal to control queue
+      when: journal_mismatch_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_mismatch_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_mismatch
+      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
+
+    # SSH error path: TCP OK, but SSH failed
+    - name: Build failure journal payload (ssh error) + mark retry
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_fail_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            {{ (banner_probe.stderr | default('') | trim) }}
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+          _needs_retry: true
+
+    - name: Publish failure journal (ssh error) to control queue
+      when: nc_probe.rc == 0 and banner_probe.rc != 0
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_fail_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_fail
+      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
+
+    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
+    - name: Compute next-attempt delay (ms) according to policy
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.set_fact:
+        next_attempt: "{{ attempt | int + 1 }}"
+        next_delay_sec: >-
+          {% if attempt | int == 1 %}
+          300
+          {% elif attempt | int == 2 %}
+          600
+          {% else %}
+          0
+          {% endif %}
+        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
+
+    # If we've reached the cap, send a final “gave up” journal and stop.
+    - name: Build final gave-up journal (max attempts reached)
+      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        journal_gaveup_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
+            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
+
+    - name: Publish final gave-up journal
+      when: journal_gaveup_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_gaveup_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_j_gaveup
+      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
+
+    - name: Stop host after final gave-up
+      when: journal_gaveup_payload is defined
+      ansible.builtin.meta: end_host
+
+    # Otherwise schedule the next attempt (only if we still have budget)
+    - name: Build delayed after-upgrade payload for next attempt
+      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        delayed_payload:
+          task_name: "afterupgrade_check"
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          attempt: "{{ next_attempt | int }}"
+          max_attempts: "{{ effective_max_attempts | int }}"
+          correlation_id: "{{ correlation_id }}"
+          original_emitted_at: "{{ original_emitted_at }}"
+          target_version: "{{ target_version_full }}"
+          current_delay_sec: "{{ next_delay_sec | int }}"
+          schema_version: 1
+
+    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
+      when: delayed_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            expiration: "{{ (next_delay_ms | int) | string }}"
+            correlation_id: "{{ correlation_id }}"
+          routing_key: "deviceconfig"
+          payload: "{{ delayed_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_pub_next
+      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
+
+    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
+      when: (_needs_retry | default(false)) | bool
+      ansible.builtin.meta: end_host

files/ansible-playbooks/indoor-restart-cloudagent.yml

@@ -0,0 +1,714 @@
+- name: Indoor DEV2 cloud-agent bounce via DEV1 → LLDP/tunnel → DEV2 (connection logic preserved; bootenv removed)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Busybox-safe PATH prefix for all remote raw calls on DEV1
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    # DEV1 credentials (stable, like rebootin222)
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    # Tunnel target DEV2 behind DEV1
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    # Temp IP we add to DEV1 so it can reach DEV2
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+
+    # DEV2 behind the tunnel (or reachable directly via LLDP 10.x)
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    # SSH options used from controller
+    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=30"
+
+    # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # ---------------- Debugging toggle ----------------
+    debugging: true
+
+    # ---------------- DEV2 connection (decided early) ----------------
+    # "tunnel" (default) or "direct_lldp"
+    dev2_conn_method: "tunnel"
+    dev2_ssh_host: ""
+    dev2_ssh_port: ""
+
+  pre_tasks:
+    # ------------------------------- Hostname sanity DEV1 -------------------------------
+    - name: Read DEV1 hostname (busybox-safe)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Debug incoming parameters and defaults
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "inventory_hostname={{ inventory_hostname }}"
+          - "rmq_host={{ rmq_host }}"
+          - "rmq_port={{ rmq_port }}"
+          - "rmq_vhost={{ rmq_vhost }}"
+          - "rmq_exchange={{ rmq_exchange }}"
+          - "control_queue={{ control_queue }}"
+
+    - name: Stop early if connected DEV1 hostname != inventory
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    # ============================ LLDP-FIRST CONNECTION DECISION ============================
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+    - name: Discover DEV2 candidate IP via LLDP on DEV1
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP (if any)
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP candidate range
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidate and classification
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP candidate IP={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP class={{ lldp_ip_class | default('none') }}"
+
+    # -------------------- CHANGE 1: override tunnel target from LLDP for 192.168.x.x --------------------
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x (for tunnel target)
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method to tunnel by default
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "tunnel"
+      changed_when: false
+
+    - name: Switch to direct LLDP mode for 10.x.x.x
+      when: (lldp_ip_class | trim) == "10"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "direct_lldp"
+      changed_when: false
+
+    - name: Debug connection method decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_method={{ dev2_conn_method }}"
+          - "lldp_dev2_ip={{ lldp_dev2_ip | default('<none>') }}"
+
+    # ============================ DIRECT LLDP AUTH (10.x) ============================
+    - name: Try DEV2 login via direct LLDP IP with 'basicpass' (10.x)
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass' for direct LLDP if previous login succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IP with 'basicpass2' (10.x, only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used is not defined)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass2' for direct LLDP if second login succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Mark DEV2 auth as NONE for direct LLDP if both attempts failed
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    - name: Set direct LLDP DEV2 SSH host/port (if auth succeeded)
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_ssh_host: "{{ lldp_dev2_ip }}"
+        dev2_ssh_port: 22
+      changed_when: false
+
+    # ============================ TUNNEL PREP (DEV1 temp IP + tunnel) ============================
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_conn_method == "tunnel" and (dev2_mac is not defined or dev2_mac | length == 0)
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+            for i in $(seq 1 25); do
+                p="$(shuf -i 20000-39999 -n 1)"
+                if command -v ss >/dev/null 2>&1; then
+                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                        echo "$p"; return 0
+                    fi
+                else
+                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                        echo "$p"; return 0
+                    fi
+                fi
+            done
+            return 1
+        }
+        pick
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Record chosen local port and create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
+
+    - name: Build path for SSH ControlMaster socket
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port → DEV2:22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+            -M -S "{{ _ctrl_sock }}" \
+            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+            "${USER}@${HOST}"
+      args:
+        executable: /bin/bash
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass' if previous login succeeded (tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login with 'basicpass2' (only if first failed, tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args:
+        executable: /bin/bash
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select 'basicpass2' if second login succeeded (tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "basicpass2"
+      changed_when: false
+
+    - name: Mark DEV2 auth as NONE if both attempts failed (tunnel)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used is not defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used: "NONE"
+      changed_when: false
+
+    - name: Set DEV2 SSH host/port for tunnel mode (if auth succeeded)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_ssh_host: "127.0.0.1"
+        dev2_ssh_port: "{{ _local_port }}"
+      changed_when: false
+
+    # -------------------- CHANGE 2: safety guard using dev2_mac (only if we actually discovered one) --------------------
+    - name: Read remote eth0 MAC via selected connection (guard ensure this is DEV2)
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE" and (dev2_mac | default('') | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "cat /sys/class/net/eth0/address 2>/dev/null || echo"
+      args:
+        executable: /bin/bash
+      register: dev2_eth0_mac_read
+      changed_when: false
+
+    - name: Abort if remote eth0 MAC != discovered DEV2 MAC
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used != "NONE" and (dev2_mac | default('') | trim | length > 0) and ((dev2_eth0_mac_read.stdout | default('') | trim | lower) != (dev2_mac | trim | lower))
+      ansible.builtin.fail:
+        msg: >
+          Safety stop: tunnel reached wrong device.
+          expected_dev2_mac={{ dev2_mac | trim }},
+          remote_eth0_mac={{ dev2_eth0_mac_read.stdout | default('') | trim }}.
+
+    # ============================ DEV2 HOSTNAME GUARD ============================
+    - name: Stop and journal if DEV2 auth failed (no passfile worked)
+      when: dev2_passfile_used == "NONE"
+      block:
+        - name: Build control queue payload for indoor aborted journal (auth failure)
+          ansible.builtin.set_fact:
+            journal_indoor_aborted:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                indoor: aborted: DEV2 auth failed (basicpass/basicpass2 did not work). conn_method={{ dev2_conn_method }}
+          delegate_to: localhost
+
+        - name: Publish indoor aborted journal (auth failure)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_indoor_aborted | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_indoor_aborted_auth_resp
+          changed_when: (rmq_journal_indoor_aborted_auth_resp.json is defined) and (rmq_journal_indoor_aborted_auth_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_indoor_aborted_auth_resp.status != 200) or
+            (rmq_journal_indoor_aborted_auth_resp.json is not defined) or
+            (not (rmq_journal_indoor_aborted_auth_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop host after DEV2 auth failure
+          ansible.builtin.meta: end_host
+
+    - name: Read DEV2 hostname via selected connection (busybox-safe)
+      when: dev2_passfile_used != "NONE"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
+      args:
+        executable: /bin/bash
+      register: dev2_host_read
+      changed_when: false
+
+    - name: Normalize hostnames for strict compare (inventory/DEV1/DEV2)
+      ansible.builtin.set_fact:
+        _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+        _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
+
+    - name: Guard DEV2 hostname must equal inventory AND DEV1 (prevents IP churn mistakes)
+      block:
+        - name: Fail if DEV2 hostname differs from inventory/DEV1
+          ansible.builtin.fail:
+            msg: >
+              Hostname mismatch: DEV2='{{ _dev2_hn }}',
+              inventory='{{ _inv_hn }}',
+              DEV1='{{ _dev1_hn }}'
+          when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
+      rescue:
+        - name: Build control queue payload for indoor aborted journal (hostname mismatch)
+          ansible.builtin.set_fact:
+            journal_indoor_aborted:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                indoor: aborted: Hostname mismatch: DEV2={{ _dev2_hn }}, inventory={{ _inv_hn }}, DEV1={{ _dev1_hn }}
+          delegate_to: localhost
+
+        - name: Publish indoor aborted journal (hostname mismatch)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_indoor_aborted | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_indoor_aborted_hn_resp
+          changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_indoor_aborted_hn_resp.status != 200) or
+            (rmq_journal_indoor_aborted_hn_resp.json is not defined) or
+            (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop host after hostname mismatch
+          ansible.builtin.meta: end_host
+
+    # ============================ JOURNAL: START ============================
+    - name: Build control queue payload for 'indoor start' journal (cloud-agent bounce)
+      ansible.builtin.set_fact:
+        journal_indoor_start:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: DEV2 reachable and hostname verified; starting cloud-agent bounce.
+            conn_method={{ dev2_conn_method }},
+            dev2={{ dev2_ssh_host }}:{{ dev2_ssh_port }}
+      delegate_to: localhost
+
+    - name: Publish 'indoor start' journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_start | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_start_resp
+      changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_start_resp.status != 200) or
+        (rmq_journal_indoor_start_resp.json is not defined) or
+        (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    # ============================ CLOUD-AGENT BOUNCE (DEV2) ============================
+    - name: Move /tmp/launchd/services/cloud-agent to /root/ on DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "set -e; PATH=/sbin:/usr/sbin:/bin:/usr/bin:\$PATH; mv -f /tmp/launchd/services/cloud-agent /root/"
+      args:
+        executable: /bin/bash
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 3
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/ on DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ dev2_ssh_host }}"
+        PORT="{{ dev2_ssh_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o AddressFamily=inet \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+          -o ConnectTimeout=30 \
+          -p "$PORT" "{{ dev2_ssh_user }}@${HOST}" \
+          "set -e; PATH=/sbin:/usr/sbin:/bin:/usr/bin:\$PATH; mv -f /root/cloud-agent /tmp/launchd/services/"
+      args:
+        executable: /bin/bash
+      register: move_back
+      changed_when: true
+
+    - name: Build 'indoor updated' journal payload (cloud-agent bounced)
+      ansible.builtin.set_fact:
+        journal_indoor_updated:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            Indoor: cloud-agent bounced successfully.
+            conn_method={{ dev2_conn_method }},
+            move_out_rc={{ move_out.rc | default('NA') }},
+            move_back_rc={{ move_back.rc | default('NA') }}
+      delegate_to: localhost
+
+    - name: Publish 'indoor updated' journal to control queue
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ journal_indoor_updated | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_indoor_updated_resp
+      changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_indoor_updated_resp.status != 200) or
+        (rmq_journal_indoor_updated_resp.json is not defined) or
+        (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    # ---------------------------- Final operator summary (one-liners) ----------------------------
+    - name: Summary key outcomes (one-liners)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_method={{ dev2_conn_method }}"
+          - "dev2_ssh_host={{ dev2_ssh_host | default('') }}"
+          - "dev2_ssh_port={{ dev2_ssh_port | default('') }}"
+          - "dev2_passfile_used={{ dev2_passfile_used }}"
+          - "dev2_hostname={{ _dev2_hn | default('') }}"
+          - "bounce_move_out_rc={{ move_out.rc | default('NA') }}"
+          - "bounce_move_back_rc={{ move_back.rc | default('NA') }}"
+
+  post_tasks:
+    - name: Cleanup (always)
+      block:
+        - ansible.builtin.debug:
+            msg: "Entering cleanup block"
+          changed_when: false
+          delegate_to: localhost
+      always:
+        - name: Close SSH ControlMaster (best-effort)
+          when: dev2_conn_method == "tunnel"
+          delegate_to: localhost
+          ansible.builtin.shell: |
+            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+          changed_when: false
+          ignore_errors: true
+
+        - name: Remove tunnel control dir (best-effort)
+          when: dev2_conn_method == "tunnel"
+          delegate_to: localhost
+          ansible.builtin.file:
+            path: "{{ _ctrl_dir | default('/tmp/none') }}"
+            state: absent
+          ignore_errors: true
+
+        - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+          when: dev2_conn_method == "tunnel"
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+          register: del_ip
+          changed_when: del_ip.rc == 0
+          failed_when: >
+            del_ip.rc != 0
+            and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+            and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
+
+    - name: Debug temp IP removal result
+      when: dev2_conn_method == "tunnel" and del_ip is defined
+      ansible.builtin.debug:
+        msg:
+          - "del_ip.rc={{ del_ip.rc | default('') }}"
+          - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
+          - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"

files/ansible-playbooks/ptsd_reacquire_attempt-old.yml

@@ -0,0 +1,759 @@
+# ptsd_reacquire_attempt.yml
+# PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check + case A cleanup
+
+- name: "PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+  pre_tasks:
+    - name: Initialize passfile facts defensively (avoid undefined vars later)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    - name: Read DEV1 hostname (busybox-safe)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop early if connected DEV1 hostname != inventory (guard)
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+ #   - name: Restart LLDP on DEV1 to refresh DEV2 management data
+ #     ansible.builtin.raw: >
+ #       {{ pathprefix }}
+ #       mv /tmp/launchd/services/lldp-server /root/lldp-server;
+ #       sleep 2;
+ #       mv /root/lldp-server /tmp/launchd/services/lldp-server;
+ #       sleep 10
+ #     ignore_errors: true
+
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port to DEV2 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable:
+          tunnel={{ dev2_passfile_used_tunnel | default('n/a') }},
+          direct={{ dev2_passfile_used_direct | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} forwarded 127.0.0.1:{{ _local_port | default('na') }}"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for verification commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR DEV2_CMD empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            *)
+              echo "ERROR unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+    - name: VERIFY DEV2 check ppp0 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show ppp0 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_ppp0_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug ppp0 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe ppp0 exists raw={{ (dev2_ppp0_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe ppp0 exists rc={{ dev2_ppp0_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show eth0.4000 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 link probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 exists raw={{ (dev2_eth04000_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 exists rc={{ dev2_eth04000_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 has IPv4?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip -4 addr show dev eth0.4000 2>/dev/null | grep -m1 'inet ' >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_has_ipv4
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 IPv4 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 has IPv4 raw={{ (dev2_eth04000_has_ipv4.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 has IPv4 rc={{ dev2_eth04000_has_ipv4.rc | default('n/a') }}"
+
+    - name: VERIFY Summarize DHCP migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        ppp0_exists: "{{ (dev2_ppp0_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_exists: "{{ (dev2_eth04000_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_has_ipv4: "{{ (dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES' }}"
+        dhcp_migrated_ok: >-
+          {{
+            ((dev2_ppp0_exists.stdout | default('') | trim) != 'YES')
+            and
+            ((dev2_eth04000_exists.stdout | default('') | trim) == 'YES')
+            and
+            ((dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES')
+          }}
+      changed_when: false
+
+    - name: VERIFY Classify migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dhcp_state: >-
+          {% if (dhcp_migrated_ok | bool) %}
+          case_a_dhcp_ok
+          {% elif (ppp0_exists | bool) and (not (eth04000_exists | bool)) %}
+          case_b_pppoe_old
+          {% elif (ppp0_exists | bool) and (eth04000_exists | bool) %}
+          case_c_mixed
+          {% else %}
+          unknown
+          {% endif %}
+      changed_when: false
+
+    - name: VERIFY Report migration state summary
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 connectivity via {{ dev2_conn_final }}"
+          - "DEV2 probes ppp0={{ ppp0_exists }} eth0.4000={{ eth04000_exists }} eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "DEV2 DHCP migrated ok={{ dhcp_migrated_ok }}"
+
+    - name: VERIFY Report why migration is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 is not confirmed as DHCP-migrated"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "Expected for success case A: ppp0 absent, eth0.4000 present, eth0.4000 has IPv4"
+          - "Observed: ppp0={{ ppp0_exists }}, eth0.4000={{ eth04000_exists }}, eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+      changed_when: false
+
+    - name: VERIFY End host if DHCP migration state is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      ansible.builtin.meta: end_host
+
+    - name: CASE A DEV2 run config-test check before reboot cancellation
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="/usr/sbin/config-test.lua -c >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: config_test_check
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A set config-test check status
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        config_test_ok: "{{ (config_test_check.stdout | default('') | trim) == 'OK' }}"
+      changed_when: false
+
+    - name: CASE A append config-test check result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            (cleanup_report | default('case A observed'))
+            ~ ', '
+            ~ (
+              'config-test -c ok'
+              if (config_test_ok | bool)
+              else 'config-test -c failed'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A Initialize cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: "case A observed"
+      changed_when: false
+
+    - name: CASE A DEV2 find if pending reboot is present
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && echo FOUND || echo NOT_FOUND"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_present
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A DEV2 cancel reboot if present
+      when: dhcp_migrated_ok | bool and (config_test_ok | bool) and (reboot_present.stdout | default('') | trim) == "FOUND"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && killall reboot 2>/dev/null || true; echo DONE"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_cancel
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append reboot cancellation result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'pending reboot found and cancelled'
+              if (reboot_present.stdout | default('') | trim) == 'FOUND'
+              else 'no pending reboot'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 touch confirming_success marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="touch /tmp/confirming_success >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: confirming_success_touch
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append confirming_success marker result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'touched /tmp/confirming_success'
+              if (confirming_success_touch.stdout | default('') | trim) == 'OK'
+              else 'failed to touch /tmp/confirming_success'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A sleep 5 seconds
+      when: dhcp_migrated_ok | bool
+      ansible.builtin.pause:
+        seconds: 5
+
+    - name: CASE A DEV2 move config test to config with dhcp
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="mv /tmp/config-test.json /tmp/config-with-dhcp.json >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: mv_config_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append config move result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'moved /tmp/config-test.json to /tmp/config-with-dhcp.json'
+              if (mv_config_result.stdout | default('') | trim) == 'OK'
+              else 'failed to move /tmp/config-test.json to /tmp/config-with-dhcp.json'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 remove ptsd inprogress marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="rm -f /tmp/ptsd.inprogress >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: rm_inprogress_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append ptsd inprogress removal result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'removed /tmp/ptsd.inprogress'
+              if (rm_inprogress_result.stdout | default('') | trim) == 'OK'
+              else 'failed to remove /tmp/ptsd.inprogress'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A final cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ cleanup_report }}"
+      changed_when: false
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup best-effort, closing tunnel, removing temp IP, removing staged passfiles"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/ptsd_reacquire_attempt.yml

@@ -0,0 +1,959 @@
+# ptsd_reacquire_attempt.yml
+# PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check + case A cleanup
+
+- name: "PTSD DHCP Reacquire Attempt | Phase 0 | Connectivity + wrapper + DHCP state check"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=15
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+  pre_tasks:
+    - name: Initialize passfile facts defensively (avoid undefined vars later)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_tunnel6: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    - name: Read DEV1 hostname (busybox-safe)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
+      register: dev1_host_read
+      changed_when: false
+
+    - name: Stop early if connected DEV1 hostname != inventory (guard)
+      ansible.builtin.meta: end_host
+      when: (dev1_host_read.stdout | trim | length > 0) and
+            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
+
+  tasks:
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+ #   - name: Restart LLDP on DEV1 to refresh DEV2 management data
+ #     ansible.builtin.raw: >
+ #       {{ pathprefix }}
+ #       mv /tmp/launchd/services/lldp-server /root/lldp-server;
+ #       sleep 2;
+ #       mv /root/lldp-server /tmp/launchd/services/lldp-server;
+ #       sleep 10
+ #     ignore_errors: true
+
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port to DEV2 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    - name: Decide if IPv6 tunnel fallback should be attempted
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        try_ipv6_tunnel: >-
+          {{
+            (lldp_dev2_ip6 | default('') | trim | length > 0)
+            and
+            (
+              (
+                (lldp_ip_class | trim) == '10'
+                and
+                (dev2_passfile_used_direct | default('NONE')) == 'NONE'
+              )
+              or
+              (
+                (lldp_ip_class | trim) == '192_168'
+                and
+                (dev2_passfile_used_tunnel | default('NONE')) == 'NONE'
+              )
+            )
+          }}
+      changed_when: false
+
+    - name: Pick a free local TCP port for the IPv6 tunnel (controller side)
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port6
+      changed_when: false
+
+    - name: Stop if no free local port was found for IPv6 tunnel
+      when: try_ipv6_tunnel | bool and (pick_port6.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for IPv6 tunnel SSH ControlMaster
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir6
+      changed_when: false
+
+    - name: Record chosen local port and build IPv6 tunnel ControlMaster socket path
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port6: "{{ pick_port6.stdout | trim }}"
+        _ctrl_dir6: "{{ mktemp_dir6.stdout | trim }}"
+        _ctrl_sock6: "{{ (mktemp_dir6.stdout | trim) }}/ssh_tunnel_ctl6"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1:local_port to DEV2 IPv6 22 via DEV1
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock6 }}" \
+          -L "127.0.0.1:{{ _local_port6 }}:[{{ lldp_dev2_ip6 }}%{{ dev1_iface }}]:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel6
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the IPv6 tunnel (nc)
+      when: try_ipv6_tunnel | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port6 }}"
+      args: { executable: /bin/bash }
+      register: nc_probe6
+      changed_when: false
+      ignore_errors: true
+
+    - name: Pick DEV2 password for root (IPv6 tunnel) try basicpass
+      when: try_ipv6_tunnel | bool and (nc_probe6.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port6 }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass6
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if IPv6 tunnel login succeeded
+      when: try_ipv6_tunnel | bool and (dev2_try_basicpass6.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel6: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through IPv6 tunnel with basicpass2 (only if first failed)
+      when: try_ipv6_tunnel | bool and (dev2_passfile_used_tunnel6 == "NONE") and (nc_probe6.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port6 }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass26
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if IPv6 tunnel login succeeded
+      when: try_ipv6_tunnel | bool and dev2_passfile_used_tunnel6 == "NONE" and (dev2_try_basicpass26.rc | default(1)) == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel6: "basicpass2"
+      changed_when: false
+
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > tunnel6 > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- elif (dev2_passfile_used_tunnel6 | default('NONE')) != 'NONE' -%}
+            tunnel6
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable:
+          tunnel={{ dev2_passfile_used_tunnel | default('n/a') }},
+          tunnel6={{ dev2_passfile_used_tunnel6 | default('n/a') }},
+          direct={{ dev2_passfile_used_direct | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "tunnel6 passfile={{ dev2_passfile_used_tunnel6 | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} forwarded 127.0.0.1:{{ _local_port | default('na') }}"
+          - "tunnel6 target=[{{ lldp_dev2_ip6 | default('') }}%{{ dev1_iface }}]:{{ dev2_port }} forwarded 127.0.0.1:{{ _local_port6 | default('na') }}"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for verification commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR DEV2_CMD empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            tunnel6)
+              PORT="{{ _local_port6 | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel6 }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            *)
+              echo "ERROR unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+    - name: VERIFY DEV2 check ppp0 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show ppp0 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_ppp0_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug ppp0 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe ppp0 exists raw={{ (dev2_ppp0_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe ppp0 exists rc={{ dev2_ppp0_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 exists?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip link show eth0.4000 >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_exists
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 link probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 exists raw={{ (dev2_eth04000_exists.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 exists rc={{ dev2_eth04000_exists.rc | default('n/a') }}"
+
+    - name: VERIFY DEV2 check eth0.4000 has IPv4?
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD="ip -4 addr show dev eth0.4000 2>/dev/null | grep -m1 'inet ' >/dev/null 2>&1 && echo YES || echo NO"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: dev2_eth04000_has_ipv4
+      changed_when: false
+      failed_when: false
+
+    - name: VERIFY Debug eth0.4000 IPv4 probe result
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - 'DEV2 probe eth0.4000 has IPv4 raw={{ (dev2_eth04000_has_ipv4.stdout | default("") ) | trim }}'
+          - "DEV2 probe eth0.4000 has IPv4 rc={{ dev2_eth04000_has_ipv4.rc | default('n/a') }}"
+
+    - name: VERIFY Summarize DHCP migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        ppp0_exists: "{{ (dev2_ppp0_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_exists: "{{ (dev2_eth04000_exists.stdout | default('') | trim) == 'YES' }}"
+        eth04000_has_ipv4: "{{ (dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES' }}"
+        dhcp_migrated_ok: >-
+          {{
+            ((dev2_ppp0_exists.stdout | default('') | trim) != 'YES')
+            and
+            ((dev2_eth04000_exists.stdout | default('') | trim) == 'YES')
+            and
+            ((dev2_eth04000_has_ipv4.stdout | default('') | trim) == 'YES')
+          }}
+      changed_when: false
+
+    - name: VERIFY Classify migration state
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dhcp_state: >-
+          {% if (dhcp_migrated_ok | bool) %}
+          case_a_dhcp_ok
+          {% elif (ppp0_exists | bool) and (not (eth04000_exists | bool)) %}
+          case_b_pppoe_old
+          {% elif (ppp0_exists | bool) and (eth04000_exists | bool) %}
+          case_c_mixed
+          {% else %}
+          unknown
+          {% endif %}
+      changed_when: false
+
+    - name: VERIFY Report migration state summary
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 connectivity via {{ dev2_conn_final }}"
+          - "DEV2 probes ppp0={{ ppp0_exists }} eth0.4000={{ eth04000_exists }} eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "DEV2 DHCP migrated ok={{ dhcp_migrated_ok }}"
+
+    - name: VERIFY Report why migration is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "DEV2 is not confirmed as DHCP-migrated"
+          - "DEV2 classified state={{ dhcp_state }}"
+          - "Expected for success case A: ppp0 absent, eth0.4000 present, eth0.4000 has IPv4"
+          - "Observed: ppp0={{ ppp0_exists }}, eth0.4000={{ eth04000_exists }}, eth0.4000_ipv4={{ eth04000_has_ipv4 }}"
+      changed_when: false
+
+    - name: VERIFY End host if DHCP migration state is not confirmed
+      when: not (dhcp_migrated_ok | bool)
+      ansible.builtin.meta: end_host
+
+    - name: CASE A DEV2 run config-test check before reboot cancellation
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="/usr/sbin/config-test.lua -c >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: config_test_check
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A set config-test check status
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        config_test_ok: "{{ (config_test_check.stdout | default('') | trim) == 'OK' }}"
+      changed_when: false
+
+    - name: CASE A append config-test check result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            (cleanup_report | default('case A observed'))
+            ~ ', '
+            ~ (
+              'config-test -c ok'
+              if (config_test_ok | bool)
+              else 'config-test -c failed'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A Initialize cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: "case A observed"
+      changed_when: false
+
+    - name: CASE A DEV2 find if pending reboot is present
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && echo FOUND || echo NOT_FOUND"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_present
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A DEV2 cancel reboot if present
+      when: dhcp_migrated_ok | bool and (config_test_ok | bool) and (reboot_present.stdout | default('') | trim) == "FOUND"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="ps w | grep eboot | grep -v grep >/dev/null 2>&1 && killall reboot 2>/dev/null || true; echo DONE"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: reboot_cancel
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append reboot cancellation result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'pending reboot found and cancelled'
+              if (reboot_present.stdout | default('') | trim) == 'FOUND'
+              else 'no pending reboot'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 touch confirming_success marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="touch /tmp/confirming_success >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: confirming_success_touch
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append confirming_success marker result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'touched /tmp/confirming_success'
+              if (confirming_success_touch.stdout | default('') | trim) == 'OK'
+              else 'failed to touch /tmp/confirming_success'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A sleep 5 seconds
+      when: dhcp_migrated_ok | bool
+      ansible.builtin.pause:
+        seconds: 5
+
+    - name: CASE A DEV2 move config test to config with dhcp
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="mv /tmp/config-test.json /tmp/config-with-dhcp.json >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: mv_config_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append config move result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'moved /tmp/config-test.json to /tmp/config-with-dhcp.json'
+              if (mv_config_result.stdout | default('') | trim) == 'OK'
+              else 'failed to move /tmp/config-test.json to /tmp/config-with-dhcp.json'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A DEV2 remove ptsd inprogress marker
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        export DEV2_CMD="rm -f /tmp/ptsd.inprogress >/dev/null 2>&1 && echo OK || echo FAIL"
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      register: rm_inprogress_result
+      changed_when: false
+      failed_when: false
+
+    - name: CASE A append ptsd inprogress removal result
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        cleanup_report: >-
+          {{
+            cleanup_report
+            ~ ', '
+            ~ (
+              'removed /tmp/ptsd.inprogress'
+              if (rm_inprogress_result.stdout | default('') | trim) == 'OK'
+              else 'failed to remove /tmp/ptsd.inprogress'
+            )
+          }}
+      changed_when: false
+
+    - name: CASE A final cumulative cleanup report
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "{{ cleanup_report }}"
+      changed_when: false
+
+    - name: CASE A publish cumulative cleanup report journal
+      when: dhcp_migrated_ok | bool
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (cleanup_report | default(''))
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup best-effort, closing tunnel, removing temp IP, removing staged passfiles"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster for IPv6 tunnel (best-effort)
+      when: _ctrl_sock6 is defined
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock6 | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove IPv6 tunnel control dir (best-effort)
+      when: _ctrl_dir6 is defined
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir6 | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/redirect_and_block.yml

@@ -0,0 +1,773 @@
+# redirect_and_block.yml — connectivity + stage block scripts
+# Phase 0: reliable DEV2 connectivity + wrapper build
+# Phase 1: copy startblock.sh and stopblock.sh to /root on DEV2
+
+- name: "Redirect and block | Connectivity + stage scripts"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+    block_src_dir: "/opt/containers/ansible-worker/app"
+    block_dst_dir: "/root"
+    block_files:
+      - "startblock.sh"
+      - "stopblock.sh"
+
+  tasks:
+    - name: Initialize passfile facts (avoid undefined vars in later templates)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    # ============================ PHASE 0: LLDP DISCOVERY (IPv4 + IPv6) ============================
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    # ============================ PHASE 0: PRIMARY PATH DECISION ============================
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    # ============================ PHASE 0: DIRECT LLDP IPv4 AUTH (10.x) ============================
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ============================
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1 local_port to DEV2 port 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: SAFETY GUARD (MAC CHECK) ============================
+    - name: Read remote eth0 MAC via tunnel (guard, sanitized)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used_tunnel }}" ssh {{ ssh_opts_common }} \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /sys/class/net/eth0/address 2>/dev/null || ip link show eth0 2>/dev/null" 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_eth0_mac_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Normalize remote eth0 MAC (extract last MAC-like token)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        remote_eth0_mac: >-
+          {{
+            (
+              (dev2_eth0_mac_raw.stdout | default('') | regex_replace('\r','')) ~ "\n" ~
+              (dev2_eth0_mac_raw.stderr | default('') | regex_replace('\r',''))
+            )
+            | regex_findall('([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})')
+            | last
+            | default('')
+            | lower
+          }}
+      changed_when: false
+
+    - name: Abort if remote eth0 MAC != discovered DEV2 MAC
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+        - (remote_eth0_mac | default('') | length) > 0
+        - (remote_eth0_mac | lower) != (dev2_mac | lower)
+      ansible.builtin.fail:
+        msg: >-
+          Safety stop: tunnel reached wrong device.
+          expected_dev2_mac={{ dev2_mac }}, remote_eth0_mac={{ remote_eth0_mac }}
+
+    # ============================ PHASE 0: FALLBACKS (only if tunnel auth failed) ============================
+    - name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if timeout 20s sshpass -f "$f" ssh {{ ssh_opts_common }} \
+              -o AddressFamily=inet \
+              -o ConnectTimeout=10 \
+              "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp4
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv4 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp4: "{{ (dev2_auth_lldp4.rc == 0) | ternary((dev2_auth_lldp4.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    - name: Copy DEV2 passfiles to DEV1 for IPv6 nested SSH (last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      ansible.builtin.copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        mode: "0600"
+      loop: "{{ dev2_passfiles }}"
+      ignore_errors: true
+
+    - name: Try DEV2 login via LLDP IPv6 (nested SSH through DEV1; last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        IP6="{{ lldp_dev2_ip6 }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+              "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+              "timeout {{ ssh_timeout }}s sshpass -f '/tmp/${f}' ssh \
+                 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                 -o PubkeyAuthentication=no \
+                 -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                 -o ConnectTimeout={{ ssh_timeout }} \
+                 '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' 'echo OK' " \
+              >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp6
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv6 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp6: "{{ (dev2_auth_lldp6.rc == 0) | ternary((dev2_auth_lldp6.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    # ============================ PHASE 0: FINAL DECISION + WRAPPER FACTS ============================
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- elif (dev2_passfile_used_lldp4 | default('NONE')) != 'NONE' -%}
+            lldp4_fallback
+          {%- elif (dev2_passfile_used_lldp6 | default('NONE')) != 'NONE' -%}
+            lldp6_via_dev1
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable: tunnel auth={{ dev2_passfile_used_tunnel | default('n/a') }},
+          direct auth={{ dev2_passfile_used_direct | default('n/a') }},
+          lldp4 auth={{ dev2_passfile_used_lldp4 | default('n/a') }},
+          lldp6 auth={{ dev2_passfile_used_lldp6 | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "lldp4 passfile={{ dev2_passfile_used_lldp4 | default('n/a') }}"
+          - "lldp6 passfile={{ dev2_passfile_used_lldp6 | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} (forwarded to 127.0.0.1:{{ _local_port | default('na') }})"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for Phase 1 commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR: DEV2_CMD is empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp4_fallback)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_lldp4 }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp6_via_dev1)
+              IP6="{{ lldp_dev2_ip6 }}"
+              F="{{ dev2_passfile_used_lldp6 }}"
+              sshpass -p "{{ dev1_pass }}" ssh {{ ssh_opts_common }} \
+                "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+                "sshpass -f '/tmp/${F}' ssh \
+                   -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                   -o PubkeyAuthentication=no \
+                   -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                   -o ConnectTimeout={{ ssh_timeout }} \
+                   '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' \
+                   '{{ pathprefix }} '"${DEV2_CMD}" 2>&1"
+              ;;
+            *)
+              echo "ERROR: unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+
+    - name: Ensure destination directory on DEV2 (via wrapper)
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='mkdir -p "{{ block_dst_dir }}"'
+        {{ dev2_exec_cmd }}
+      args: { executable: /bin/bash }
+      changed_when: false
+
+    - name: Copy block scripts controller to DEV2 direct (LLDP direct or LLDP4 fallback)
+      when: dev2_conn_final in ["direct_lldp", "lldp4_fallback"]
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        if [ "{{ dev2_conn_final }}" = "direct_lldp" ]; then
+          HOST="{{ lldp_dev2_ip }}"
+          PASSFILE="{{ dev2_passfile_used_direct }}"
+        else
+          HOST="{{ lldp_dev2_ip }}"
+          PASSFILE="{{ dev2_passfile_used_lldp4 }}"
+        fi
+        sshpass -f "$PASSFILE" scp {{ ssh_opts_common }} -P "{{ dev2_port }}" \
+          "{{ block_src_dir }}/{{ item }}" \
+          "{{ dev2_ssh_user }}@${HOST}:{{ block_dst_dir }}/{{ item }}"
+        sshpass -f "$PASSFILE" ssh {{ ssh_opts_common }} -p "{{ dev2_port }}" \
+          "{{ dev2_ssh_user }}@${HOST}" "chmod 0755 {{ block_dst_dir }}/{{ item }}"
+      args: { executable: /bin/bash }
+      loop: "{{ block_files }}"
+      changed_when: true
+
+    - name: Copy block scripts controller to DEV1 staging (for tunnel or LLDP6 via DEV1)
+      when: dev2_conn_final in ["tunnel", "lldp6_via_dev1"]
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        sshpass -p "{{ dev1_pass }}" scp {{ ssh_opts_common }} \
+          "{{ block_src_dir }}/{{ item }}" \
+          "{{ dev1_user }}@{{ inventory_hostname }}:/tmp/{{ item }}"
+      args: { executable: /bin/bash }
+      loop: "{{ block_files }}"
+      changed_when: true
+
+    - name: Stage DEV2 passfile on DEV1 for DEV1 to DEV2 scp
+      when: dev2_conn_final in ["tunnel", "lldp6_via_dev1"]
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PASSFILE="{{ dev2_passfile_used_tunnel if dev2_conn_final == 'tunnel' else dev2_passfile_used_lldp6 }}"
+        sshpass -p "{{ dev1_pass }}" scp {{ ssh_opts_common }} \
+          "$PASSFILE" \
+          "{{ dev1_user }}@{{ inventory_hostname }}:/tmp/ptsd_passfile_dev2"
+      args: { executable: /bin/bash }
+      changed_when: true
+
+    - name: Copy block scripts DEV1 to DEV2 (tunnel)
+      when: dev2_conn_final == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (PASSFILE="/tmp/ptsd_passfile_dev2";
+         for f in {{ block_files | join(' ') }}; do
+           sshpass -f "$PASSFILE" scp {{ ssh_opts_common }} -P {{ dev2_port }} "/tmp/$f" "{{ dev2_ssh_user }}@{{ dev2_host }}:{{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+           sshpass -f "$PASSFILE" ssh {{ ssh_opts_common }} -p {{ dev2_port }} "{{ dev2_ssh_user }}@{{ dev2_host }}" "chmod 0755 {{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+         done;
+         exit 0)
+      register: dev1_to_dev2_copy
+      changed_when: true
+      failed_when: dev1_to_dev2_copy.rc != 0
+
+    - name: Copy block scripts DEV1 to DEV2 (LLDP6 via DEV1)
+      when: dev2_conn_final == "lldp6_via_dev1"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        (PASSFILE="/tmp/ptsd_passfile_dev2";
+         HOST6="{{ lldp_dev2_ip6 }}";
+         for f in {{ block_files | join(' ') }}; do
+           sshpass -f "$PASSFILE" scp {{ ssh_opts_common }} -6 -P {{ dev2_port }} "/tmp/$f" "{{ dev2_ssh_user }}@${HOST6}:{{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+           sshpass -f "$PASSFILE" ssh {{ ssh_opts_common }} -6 -p {{ dev2_port }} "{{ dev2_ssh_user }}@${HOST6}" "chmod 0755 {{ block_dst_dir }}/$f" >/dev/null 2>&1 || exit 1;
+         done;
+         exit 0)
+      register: dev1_to_dev2_copy6
+      changed_when: true
+      failed_when: dev1_to_dev2_copy6.rc != 0
+
+    - name: Make block scripts executable on DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='chmod 0755 /root/startblock.sh /root/stopblock.sh'
+        {{ dev2_exec_cmd }}
+      args:
+        executable: /bin/bash
+      changed_when: true      
+
+    - name: Publish redirect and block deployment journal
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': 'redirect and block script deployed'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Set NetBox custom field indoor_rnb -> deployed
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_rnb',
+            'task_result': 'deployed'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup: closing tunnel, removing temp IP, removing staged passfiles (best-effort)"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/redirect_and_block_remove.yml

@@ -0,0 +1,704 @@
+# redirect_and_block_removal.yml — connectivity + stage block scripts removal
+# Phase 0: reliable DEV2 connectivity + wrapper build
+# Phase 1: run twice and remove the scripts
+
+- name: "Redirect and block | Connectivity + stage scripts"
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
+
+
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+    dev1_iface: "br-wan"
+
+    arping_iface: "eth0"
+
+    dev2_ssh_user: "root"
+    dev2_passfiles:
+      - "basicpass"
+      - "basicpass2"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+      -o ConnectionAttempts=1
+      -o LogLevel=ERROR
+
+    debugging: true
+    ssh_timeout: 30
+
+    block_src_dir: "/opt/containers/ansible-worker/app"
+    block_dst_dir: "/root"
+    block_files:
+      - "startblock.sh"
+      - "stopblock.sh"
+
+  tasks:
+    - name: Initialize passfile facts (avoid undefined vars in later templates)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "NONE"
+        dev2_passfile_used_tunnel: "NONE"
+        dev2_passfile_used_lldp4: "NONE"
+        dev2_passfile_used_lldp6: "NONE"
+      changed_when: false
+
+    # ============================ PHASE 0: LLDP DISCOVERY (IPv4 + IPv6) ============================
+    - name: Compute hostname digits key for LLDP lookup (DEV2)
+      ansible.builtin.set_fact:
+        dev2_lldp_digits: "{{ (inventory_hostname | string) | regex_replace('[^0-9]', '') }}"
+      changed_when: false
+
+    - name: Discover DEV2 candidate IPv4 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 10 \
+          | grep address \
+          | grep -vE 'subtype|ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Discover DEV2 candidate IPv6 via LLDP on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        DIGITS="{{ dev2_lldp_digits }}";
+        cat /var/run/lldp_server.json 2>/dev/null \
+          | grep "${DIGITS}" -A 15 \
+          | grep 'address_ipv6' \
+          | awk -F'"' '{ print $4 }' \
+          | head -n1
+      register: dev2_lldp_ip6_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Capture LLDP-derived DEV2 IP facts
+      ansible.builtin.set_fact:
+        lldp_dev2_ip: "{{ (dev2_lldp_ip_raw.stdout | default('')) | trim }}"
+        lldp_dev2_ip6: "{{ (dev2_lldp_ip6_raw.stdout | default('')) | trim }}"
+      changed_when: false
+
+    - name: Classify LLDP IPv4 candidate
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        lldp_ip_class: >-
+          {% set ip = (lldp_dev2_ip | default('')) %}
+          {% if ip == '' %}none
+          {% elif ip.startswith('10.') %}10
+          {% elif ip.startswith('192.168.') %}192_168
+          {% else %}other{% endif %}
+      changed_when: false
+
+    - name: Debug LLDP candidates
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "LLDP digits={{ dev2_lldp_digits | default('') }}"
+          - "LLDP IPv4 candidate={{ lldp_dev2_ip | default('<none>') }}"
+          - "LLDP IPv4 class={{ lldp_ip_class | default('none') }}"
+          - "LLDP IPv6 candidate={{ lldp_dev2_ip6 | default('<none>') }}"
+
+    # ============================ PHASE 0: PRIMARY PATH DECISION ============================
+    - name: Override dev2_host from LLDP when candidate is 192.168.x.x
+      when: (lldp_ip_class | trim) == "192_168" and (lldp_dev2_ip | trim | length > 0)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_host: "{{ lldp_dev2_ip | trim }}"
+      changed_when: false
+
+    - name: Set connection method initial (direct_lldp if 10.x else tunnel)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_method: "{{ 'direct_lldp' if ((lldp_ip_class | trim) == '10') else 'tunnel' }}"
+      changed_when: false
+
+    # ============================ PHASE 0: DIRECT LLDP IPv4 AUTH (10.x) ============================
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass
+      when: dev2_conn_method == "direct_lldp"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_lldp_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login via direct LLDP IPv4 (10.x) with basicpass2 (only if first failed)
+      when: dev2_conn_method == "direct_lldp" and (dev2_passfile_used_direct == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_lldp_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 for direct LLDP if succeeded
+      when: dev2_conn_method == "direct_lldp" and dev2_passfile_used_direct == "NONE" and dev2_lldp_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_direct: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: TUNNEL PREP (DEV1 temp IP + ARP + tunnel) ============================
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (if MAC discovered; tolerate 'File exists')
+      when: dev2_conn_method == "tunnel" and (dev2_mac | default('') | length > 0)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Refresh ARP (best-effort)
+      when: dev2_conn_method == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3
+      changed_when: false
+      failed_when: false
+
+    - name: Pick a free local TCP port for the tunnel (controller side)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        pick() {
+          for i in $(seq 1 25); do
+            p="$(shuf -i 20000-39999 -n 1)"
+            if command -v ss >/dev/null 2>&1; then
+              if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
+                echo "$p"; return 0
+              fi
+            else
+              if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
+                echo "$p"; return 0
+              fi
+            fi
+          done
+          return 1
+        }
+        pick
+      args: { executable: /bin/bash }
+      register: pick_port
+      changed_when: false
+
+    - name: Stop if no free local port was found
+      when: dev2_conn_method == "tunnel" and (pick_port.stdout | trim | length) == 0
+      ansible.builtin.meta: end_host
+
+    - name: Create control dir for SSH ControlMaster
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: "mktemp -d"
+      args: { executable: /bin/bash }
+      register: mktemp_dir
+      changed_when: false
+
+    - name: Record chosen local port and build ControlMaster socket path
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _local_port: "{{ pick_port.stdout | trim }}"
+        _ctrl_dir: "{{ mktemp_dir.stdout | trim }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | trim) }}/ssh_tunnel_ctl"
+      changed_when: false
+
+    - name: Start SSH ControlMaster and forward 127.0.0.1 local_port to DEV2 port 22 via DEV1
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      register: start_tunnel
+      changed_when: true
+
+    - name: Probe TCP reachability to DEV2 through the tunnel (nc)
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        nc -z -w5 127.0.0.1 "{{ _local_port }}"
+      args: { executable: /bin/bash }
+      register: nc_probe
+      changed_when: false
+      ignore_errors: true
+
+    - name: Stop if tunnel TCP probe failed
+      when: dev2_conn_method == "tunnel" and nc_probe.rc != 0
+      ansible.builtin.meta: end_host
+
+    - name: Pick DEV2 password for root (tunnel) try basicpass
+      when: dev2_conn_method == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_try_basicpass.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass"
+      changed_when: false
+
+    - name: Try DEV2 login through tunnel with basicpass2 (only if first failed)
+      when: dev2_conn_method == "tunnel" and (dev2_passfile_used_tunnel == "NONE")
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        PORT="{{ _local_port }}"
+        sshpass -f basicpass2 ssh {{ ssh_opts_common }} \
+          -o AddressFamily=inet \
+          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_try_basicpass2
+      changed_when: false
+      ignore_errors: true
+
+    - name: Select basicpass2 if tunnel login succeeded
+      when: dev2_conn_method == "tunnel" and dev2_passfile_used_tunnel == "NONE" and dev2_try_basicpass2.rc == 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_tunnel: "basicpass2"
+      changed_when: false
+
+    # ============================ PHASE 0: SAFETY GUARD (MAC CHECK) ============================
+    - name: Read remote eth0 MAC via tunnel (guard, sanitized)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used_tunnel }}" ssh {{ ssh_opts_common }} \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /sys/class/net/eth0/address 2>/dev/null || ip link show eth0 2>/dev/null" 2>&1
+      args: { executable: /bin/bash }
+      register: dev2_eth0_mac_raw
+      changed_when: false
+      failed_when: false
+
+    - name: Normalize remote eth0 MAC (extract last MAC-like token)
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        remote_eth0_mac: >-
+          {{
+            (
+              (dev2_eth0_mac_raw.stdout | default('') | regex_replace('\r','')) ~ "\n" ~
+              (dev2_eth0_mac_raw.stderr | default('') | regex_replace('\r',''))
+            )
+            | regex_findall('([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})')
+            | last
+            | default('')
+            | lower
+          }}
+      changed_when: false
+
+    - name: Abort if remote eth0 MAC != discovered DEV2 MAC
+      when:
+        - dev2_conn_method == "tunnel"
+        - (dev2_passfile_used_tunnel | default('NONE')) != "NONE"
+        - (dev2_mac | default('') | length) > 0
+        - (remote_eth0_mac | default('') | length) > 0
+        - (remote_eth0_mac | lower) != (dev2_mac | lower)
+      ansible.builtin.fail:
+        msg: >-
+          Safety stop: tunnel reached wrong device.
+          expected_dev2_mac={{ dev2_mac }}, remote_eth0_mac={{ remote_eth0_mac }}
+
+    # ============================ PHASE 0: FALLBACKS (only if tunnel auth failed) ============================
+    - name: Try DEV2 login via LLDP IPv4 10.x (fallback if tunnel auth failed)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        HOST="{{ lldp_dev2_ip }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if timeout 20s sshpass -f "$f" ssh {{ ssh_opts_common }} \
+              -o AddressFamily=inet \
+              -o ConnectTimeout=10 \
+              "{{ dev2_ssh_user }}@${HOST}" echo OK >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp4
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv4 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - (lldp_dev2_ip | default('')) is match('^10\\.')
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp4: "{{ (dev2_auth_lldp4.rc == 0) | ternary((dev2_auth_lldp4.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    - name: Copy DEV2 passfiles to DEV1 for IPv6 nested SSH (last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      ansible.builtin.copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        mode: "0600"
+      loop: "{{ dev2_passfiles }}"
+      ignore_errors: true
+
+    - name: Try DEV2 login via LLDP IPv6 (nested SSH through DEV1; last resort)
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        set -e
+        IP6="{{ lldp_dev2_ip6 }}"
+        for f in {{ dev2_passfiles | join(' ') }}; do
+          if sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+              "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
+              "timeout {{ ssh_timeout }}s sshpass -f '/tmp/${f}' ssh \
+                 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                 -o PubkeyAuthentication=no \
+                 -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                 -o ConnectTimeout={{ ssh_timeout }} \
+                 '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' 'echo OK' " \
+              >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        echo "NONE"; exit 1
+      args: { executable: /bin/bash }
+      register: dev2_auth_lldp6
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Record LLDP IPv6 fallback decision
+      when:
+        - dev2_conn_method == "tunnel"
+        - dev2_passfile_used_tunnel == "NONE"
+        - dev2_passfile_used_lldp4 == "NONE"
+        - (lldp_dev2_ip6 | default('') | length) > 0
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_passfile_used_lldp6: "{{ (dev2_auth_lldp6.rc == 0) | ternary((dev2_auth_lldp6.stdout | trim), 'NONE') }}"
+      changed_when: false
+
+    # ============================ PHASE 0: FINAL DECISION + WRAPPER FACTS ============================
+    - name: Decide final DEV2 connection mode (direct_lldp > tunnel > lldp4 > lldp6)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_conn_final: >-
+          {%- if dev2_conn_method == 'direct_lldp' and (dev2_passfile_used_direct | default('NONE')) != 'NONE' -%}
+            direct_lldp
+          {%- elif dev2_conn_method == 'tunnel' and (dev2_passfile_used_tunnel | default('NONE')) != 'NONE' -%}
+            tunnel
+          {%- elif (dev2_passfile_used_lldp4 | default('NONE')) != 'NONE' -%}
+            lldp4_fallback
+          {%- elif (dev2_passfile_used_lldp6 | default('NONE')) != 'NONE' -%}
+            lldp6_via_dev1
+          {%- else -%}
+            none
+          {%- endif -%}
+      changed_when: false
+
+    - name: Abort if all DEV2 connection methods failed
+      when: dev2_conn_final == "none"
+      ansible.builtin.fail:
+        msg: >
+          DEV2 unreachable: tunnel auth={{ dev2_passfile_used_tunnel | default('n/a') }},
+          direct auth={{ dev2_passfile_used_direct | default('n/a') }},
+          lldp4 auth={{ dev2_passfile_used_lldp4 | default('n/a') }},
+          lldp6 auth={{ dev2_passfile_used_lldp6 | default('n/a') }}.
+
+    - name: Debug final connectivity decision
+      when: debugging | bool
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev2_conn_final={{ dev2_conn_final }}"
+          - "tunnel passfile={{ dev2_passfile_used_tunnel | default('n/a') }}"
+          - "direct passfile={{ dev2_passfile_used_direct | default('n/a') }}"
+          - "lldp4 passfile={{ dev2_passfile_used_lldp4 | default('n/a') }}"
+          - "lldp6 passfile={{ dev2_passfile_used_lldp6 | default('n/a') }}"
+          - "tunnel target={{ dev2_host }}:{{ dev2_port }} (forwarded to 127.0.0.1:{{ _local_port | default('na') }})"
+          - "lldp ipv4={{ lldp_dev2_ip | default('') }}"
+          - "lldp ipv6={{ lldp_dev2_ip6 | default('') }}"
+
+    - name: Build DEV2 exec wrapper (controller-side) for Phase 1 commands
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev2_exec_cmd: |
+          set -e
+          MODE="{{ dev2_conn_final }}"
+          if [ -z "${DEV2_CMD:-}" ]; then
+            echo "ERROR: DEV2_CMD is empty" >&2
+            exit 2
+          fi
+
+          case "$MODE" in
+            tunnel)
+              PORT="{{ _local_port | default('') }}"
+              PASS="{{ dev2_passfile_used_tunnel }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            direct_lldp)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_direct }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp4_fallback)
+              HOST="{{ lldp_dev2_ip }}"
+              PASS="{{ dev2_passfile_used_lldp4 }}"
+              sshpass -f "$PASS" ssh {{ ssh_opts_common }} \
+                -o AddressFamily=inet \
+                "{{ dev2_ssh_user }}@${HOST}" \
+                "{{ pathprefix }} ${DEV2_CMD}" 2>&1
+              ;;
+            lldp6_via_dev1)
+              IP6="{{ lldp_dev2_ip6 }}"
+              F="{{ dev2_passfile_used_lldp6 }}"
+              sshpass -p "{{ dev1_pass }}" ssh {{ ssh_opts_common }} \
+                "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+                "sshpass -f '/tmp/${F}' ssh \
+                   -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+                   -o PubkeyAuthentication=no \
+                   -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                   -o ConnectTimeout={{ ssh_timeout }} \
+                   '{{ dev2_ssh_user }}@\[${IP6}%{{ dev1_iface }}\]' \
+                   '{{ pathprefix }} '"${DEV2_CMD}" 2>&1"
+              ;;
+            *)
+              echo "ERROR: unknown MODE=$MODE" >&2
+              exit 3
+              ;;
+          esac
+      changed_when: false
+
+
+    ##########################################################################
+    # PHASE 1: ROLLBACK (execute stop, remove scripts, journal + custom field)
+    ##########################################################################
+
+    - name: Run stopblock twice with 2s pause
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='/root/stopblock.sh >/dev/null 2>&1 || true; sleep 2; /root/stopblock.sh >/dev/null 2>&1 || true'
+        {{ dev2_exec_cmd }}
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Remove block scripts from DEV2
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        export DEV2_CMD='rm -f /root/stopblock.sh /root/startblock.sh'
+        {{ dev2_exec_cmd }}
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Publish redirect and block removal journal
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': 'redirect and block removed'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: Set NetBox custom field indoor_rnb to cleared
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_rnb',
+            'task_result': 'cleared'
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+
+  post_tasks:
+    - name: Cleanup note
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "Cleanup: closing tunnel, removing temp IP, removing staged passfiles (best-effort)"
+      changed_when: false
+
+    - name: Close SSH ControlMaster (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.shell: |
+        ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
+      args: { executable: /bin/bash }
+      changed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      delegate_to: localhost
+      ansible.builtin.file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove staged passfiles from DEV1 (best-effort)
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        rm -f /tmp/basicpass /tmp/basicpass2 2>/dev/null || true
+      changed_when: false
+      failed_when: false
+
+    - name: Remove temporary IP on DEV1 (tolerate 'Cannot assign requested address')
+      when: dev2_conn_method | default('') == "tunnel"
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: del_ip
+      changed_when: del_ip.rc == 0
+      failed_when: >
+        del_ip.rc != 0
+        and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
+        and ('Cannot assign requested address' not in (del_ip.stderr | default('')))

files/ansible-playbooks/restart-cloud-agent.yml

@@ -0,0 +1,25 @@
+---
+- name: Temporarily move cloud-agent and restore it
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Move /tmp/launchd/services/cloud-agent to /root/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /tmp/launchd/services/cloud-agent /root/
+      register: move_out
+      changed_when: true
+
+    - name: Wait 3s before restoring
+      ansible.builtin.pause:
+        seconds: 3
+
+    - name: Move /root/cloud-agent back to /tmp/launchd/services/
+      ansible.builtin.raw: |
+        set -e
+        PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH
+        mv -f /root/cloud-agent /tmp/launchd/services/
+      register: move_back
+      changed_when: true

files/ansible-playbooks/sot-updater-iponly.yml

@@ -0,0 +1,212 @@
+# sot-updater-iponly.yml — NetBox IP sync ONLY via nb_onedevice_update.py (no device scan)
+
+- name: NetBox IP sync only (Cloud → NetBox via nb_onedevice_update.py); do not scan devices
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # Wrapper mode:
+    #   false (default): standalone behavior (may end_host)
+    #   true: composable behavior for sot-updater-wrapper.yml (no end_host)
+    sot_wrapper_mode: false
+
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=15
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }}"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Detect cloud failure"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      delegate_facts: true
+      vars:
+        _out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        _lines: >-
+          {{ _out.split('\n') | map('trim') | list }}
+        _cloud_fail_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
+        _error_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+') | list | last | default('')) }}
+        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
+        _cloud_error_type: >-
+          {%- if _error_line is search('network error')
+                and _error_line is search('500 error responses') -%}
+            cloud_http_500
+          {%- elif (_cloud_fail_line | length) > 0 -%}
+            cloud_logic
+          {%- elif _rc_is_3 -%}
+            cloud_other
+          {%- else -%}
+            none
+          {%- endif -%}
+        _cloud_retryable: >-
+          {{ _cloud_error_type not in ['cloud_http_500'] }}
+      set_fact:
+        cloud_fail_line: "{{ _cloud_fail_line }}"
+        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"
+        cloud_error_type: "{{ _cloud_error_type }}"
+        cloud_retryable: "{{ _cloud_retryable }}"
+
+    # --- Extract upgrade_cmd line if any ---
+    - name: "NB preflight | Extract upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\\s*upgrade_cmd\\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Parse upgrade_cmd value"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args: { executable: /bin/bash }
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd='{{ nb_upgrade_cmd }}' (len={{ nb_upgrade_cmd|length }})"
+
+    - name: "NB preflight | Parse OK line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      delegate_facts: true
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=') 
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc|default(1)) == 0 and (nb_ok_line|length)>0 }}"
+        nb_dev: "{{ (nb_tokens|first|default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip','') }}"
+        nb_fw: "{{ nb_kv.get('fw','') }}"
+        nb_node: "{{ nb_kv.get('node','') }}"
+        nb_sector: "{{ nb_kv.get('sector','') }}"
+        nb_small: "{{ nb_kv.get('small','') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    - name: "NB preflight | Detect IP change"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      delegate_facts: true
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      debug:
+        msg:
+          - "Preflight verdict:"
+          - "Cloud fail: {{ cloud_bad|default(false) }}"
+          - "IP changed: {{ nb_ip_changed|default(false) }}"
+          - "Reason: {{ nb_change_reason|default('none') }}"
+
+    - name: "NB preflight | Pause 1s if OK"
+      when:
+        - nb_script.stat.exists
+        - not ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
+      delegate_to: localhost
+      pause:
+        seconds: 1
+
+  tasks:
+    - name: "IP-only | Stop after NetBox sync (no device scan)"
+      when: not (sot_wrapper_mode | default(false))
+      meta: end_host

files/ansible-playbooks/sot-updater-upgradecmd.yml

@@ -0,0 +1,752 @@
+# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
+
+- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+    dev1_user: "root"
+    dev1_pass: "wavewave"
+    ssh_timeout: 30
+
+    dev2_host: "192.168.1.1"
+    dev2_port: 22
+    dev2_ssh_user: "root"
+    dev2_passfiles: [ "basicpass", "basicpass2" ]
+
+    dev2_side_ip: "192.168.1.11/24"
+    dev1_iface: "br-wan"
+    arping_iface: "eth0"
+    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
+
+    ssh_opts_common: >-
+      -o PreferredAuthentications=password
+      -o PubkeyAuthentication=no
+      -o StrictHostKeyChecking=no
+      -o UserKnownHostsFile=/dev/null
+      -o NumberOfPasswordPrompts=1
+      -o ConnectTimeout=30
+
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # === NEW (surgical): requeue attempt counter from environment, default 0 ===
+    requeue_attempt: "{{ (lookup('env','REQUEUE_ATTEMPT') | default('0', true)) | int }}"
+
+  pre_tasks:
+    - name: "NB preflight | Verify script exists"
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
+      register: nb_script
+
+    - name: "NB preflight | Abort softly if script missing (path typo?)"
+      when: not nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
+          - "Tip: adjust chdir/path or script name."
+
+    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }} (chatty)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      environment:
+        PYTHONUNBUFFERED: "1"
+      args:
+        chdir: "/opt/containers/ansible-worker/app"
+        executable: /bin/bash
+      shell: |
+        set -o pipefail
+        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
+      register: nb_preflight
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Show results"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "rc={{ nb_preflight.rc }}"
+          - "stdout_lines:"
+          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
+          - "stderr_lines:"
+          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
+          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
+
+    # === HARD STOP ON CLOUD FAILURE (immediately after preflight) ===
+    - name: "NB preflight | Detect cloud failure (rc==3 OR 'FAIL Cloud' line)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        _lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r',''))
+             | split('\n') | map('trim') | list }}
+        _cloud_fail_line: >-
+          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
+        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
+      ansible.builtin.set_fact:
+        cloud_fail_line: "{{ _cloud_fail_line }}"
+        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"
+
+    - name: "NB preflight | Journal + STOP (cloud unavailable)"
+      when:
+        - nb_script.stat.exists
+        - cloud_bad | default(false)
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': (
+              'preflight abort: cloud unavailable; closing without requeue. '
+              ~ (cloud_fail_line | default(''))
+              ~ ' rc=' ~ ((nb_preflight.rc | default('')) | string)
+            )
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_cloud_unavail
+      changed_when: (rmq_cloud_unavail.json is defined) and (rmq_cloud_unavail.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: "NB preflight | Stop host due to cloud unavailability (no requeue)"
+      when:
+        - nb_script.stat.exists
+        - cloud_bad | default(false)
+      ansible.builtin.meta: end_host
+    # === END HARD STOP ===
+
+    # --- Debug-only: capture the exact upgrade_cmd line and stop the play ---
+    - name: "NB preflight | Extract exact upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        nb_upgrade_line: >-
+          {{
+            (
+              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
+              | select('match', '^NB:\s*upgrade_cmd\s*=')
+              | list | first
+            ) | default('')
+          }}
+
+    - name: "NB preflight | Show captured upgrade_cmd line"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "upgrade_cmd line (exact): {{ nb_upgrade_line | default('<not found>') }}"
+          - "found? {{ (nb_upgrade_line | length) > 0 }}"
+
+    - name: "NB preflight | Parse value after '=' via awk"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
+      args:
+        executable: /bin/bash
+      register: up_cmd_sh
+      changed_when: false
+
+    - name: "NB preflight | Set and show parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      set_fact:
+        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"
+
+    - name: "NB preflight | Debug parsed upgrade_cmd"
+      when: nb_upgrade_line | length > 0
+      delegate_to: localhost
+      debug:
+        msg:
+          - "upgrade_cmd value: '{{ nb_upgrade_cmd }}'"
+          - "len={{ nb_upgrade_cmd | length }}"
+
+    # --- Parse the OK line robustly (token-based) ---
+    - name: "NB preflight | Parse OK line (token)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        nb_lines: >-
+          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
+        nb_ok_line: >-
+          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
+        nb_tokens: >-
+          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
+        nb_kv: >-
+          {{
+            dict(
+              nb_tokens
+              | select('match', '^[a-zA-Z_]+=')
+              | map('split', '=', 1)
+              | map('list')
+            )
+          }}
+      set_fact:
+        nb_ok: "{{ (nb_preflight.rc | default(1)) == 0 and (nb_ok_line | length) > 0 }}"
+        nb_dev: "{{ (nb_tokens | first | default('')) if (nb_tokens|length>0) else '' }}"
+        nb_ip: "{{ nb_kv.get('ip', '') }}"
+        nb_fw: "{{ nb_kv.get('fw', '') }}"
+        nb_node: "{{ nb_kv.get('node', '') }}"
+        nb_sector: "{{ nb_kv.get('sector', '') }}"
+        nb_small: "{{ nb_kv.get('small', '') }}"
+        nb_ok_line: "{{ nb_ok_line }}"
+
+    # --- Detect "cloud vs NetBox (before update) was different" (regex-free, robust)
+    - name: "NB preflight | Detect whether IP changed (pre-update)"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      vars:
+        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
+        reason: >-
+          {%- if 'IP: moving ' in out -%}moving
+          {%- elif 'IP: create new ' in out -%}create new
+          {%- elif 'IP: pruning stale ' in out -%}pruning stale
+          {%- else -%}none{%- endif -%}
+      set_fact:
+        nb_ip_changed: "{{ reason != 'none' }}"
+        nb_change_reason: "{{ reason }}"
+
+    - name: "NB preflight | Verdict"
+      when: nb_script.stat.exists
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "NB preflight verdict: {{ 'IP CHANGED (will requeue)' if nb_ip_changed else 'IP SAME (continue)' }}"
+          - "Reason: {{ nb_change_reason }}"
+
+    # --- If IP changed → publish a 3s delayed 'sot-updater' and stop this host ---
+    - name: "NB preflight | Publish delayed requeue (3s) and stop"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+        - (requeue_attempt | int) < 3              # <— NEW: limit to 3 tries
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            # === NEW (surgical): 20s delay instead of 3s ===
+            headers: { x-delay: 20000 }
+          routing_key: "deviceconfig"
+          # === NEW (surgical): include attempt counter in payload ===
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'sot-updater',
+            'attempt': ((requeue_attempt | int) + 1)
+          } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_requeue
+      changed_when: false
+      failed_when: false
+
+    - name: "NB preflight | Log requeue publish response"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+        - (requeue_attempt | int) < 3              # <— NEW: only if we actually published
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "RMQ publish URL: http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+          - "HTTP status: {{ rmq_requeue.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_requeue.json | default('<none>') }}"
+          - "Raw content: {{ rmq_requeue.content | default('<none>') }}"
+
+    - name: "NB preflight | Stop further tasks for this host"
+      when:
+        - nb_script.stat.exists
+        - nb_ip_changed | default(false)
+        - (nb_preflight.rc | default(1)) == 0      # <— extra safety
+      meta: end_host
+
+    # --- If IP did not change → optional 1s pause, then continue normally ---
+    - name: "NB preflight | Pause 1s"
+      when:
+        - nb_script.stat.exists
+        - not (nb_ip_changed | default(false))
+      delegate_to: localhost
+      ansible.builtin.pause:
+        seconds: 1
+
+  tasks:
+    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Mode: report-only (soft-fail; publish journals on failures)"
+
+    # ----------------------- Temp IP on DEV1 -----------------------
+    - name: Add temporary IP on DEV1
+      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      register: add_ip
+      failed_when: false
+      ignore_errors: true
+
+    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
+    - name: Add temporary IP on DEV1 (tolerate 'File exists')
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
+      register: add_ip
+      changed_when: add_ip.rc == 0
+      failed_when: >
+        add_ip.rc != 0
+        and ('File exists' not in (add_ip.stdout | default('')))
+        and ('File exists' not in (add_ip.stderr | default('')))
+
+    - name: Debug result of adding temp IP to DEV1
+      ansible.builtin.debug:
+        msg:
+          - "add_ip.rc={{ add_ip.rc | default('') }}"
+          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
+          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
+
+    # ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
+    - name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
+      register: dev2_mac_scan
+      changed_when: false
+      failed_when: false
+
+    - name: Capture discovered DEV2 MAC (if any)
+      ansible.builtin.set_fact:
+        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
+      changed_when: false
+
+    - name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
+      register: dev2_arp_del
+      changed_when: false
+      failed_when: false
+
+    - name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
+      when: dev2_mac is defined and dev2_mac | length > 0
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
+      register: dev2_arp_add
+      changed_when: dev2_arp_add.rc == 0
+      failed_when: >
+        dev2_arp_add.rc != 0
+        and ('File exists' not in (dev2_arp_add.stdout | default('')))
+        and ('File exists' not in (dev2_arp_add.stderr | default('')))
+
+    - name: Debug ARP action summary on DEV1
+      ansible.builtin.debug:
+        msg:
+          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
+          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
+          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
+          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
+
+    - name: Note skipping static ARP add (no MAC discovered)
+      when: dev2_mac is not defined or dev2_mac | length == 0
+      ansible.builtin.debug:
+        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
+
+    # ---------------------------- ARP refresh ----------------------------
+    - name: Refresh ARP #1
+      ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Refresh ARP #1
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Local tunnel prep -----------------------
+    - name: Pick a free local TCP port for the tunnel
+      delegate_to: localhost
+      shell: |
+        for i in $(seq 1 50); do
+          p="$(shuf -i 20000-39999 -n 1)"
+          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
+        done
+        exit 1
+      register: pick_port
+      failed_when: false
+
+    - name: Record chosen port and create control dir
+      delegate_to: localhost
+      shell: "mktemp -d"
+      register: mktemp_dir
+      failed_when: false
+
+    - name: Set facts for tunnel paths
+      delegate_to: localhost
+      set_fact:
+        _local_port: "{{ (pick_port.stdout | default('') | trim) }}"
+        _ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
+        _ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
+
+    # ----------------------- AIRPING #2 -----------------------
+    - name: Refresh ARP #2
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- Start tunnel -----------------------
+    - name: Start SSH tunnel via DEV1
+      delegate_to: localhost
+      shell: |
+        set -e
+        USER="{{ dev1_user }}"
+        HOST="{{ ansible_host | default(inventory_hostname) }}"
+        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
+          -M -S "{{ _ctrl_sock }}" \
+          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
+          "${USER}@${HOST}"
+      args: { executable: /bin/bash }
+      failed_when: false
+      ignore_errors: true
+
+    - name: Probe tunnel
+      delegate_to: localhost
+      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
+      register: nc_probe
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set tunnel_ok
+      delegate_to: localhost
+      set_fact:
+        tunnel_ok: "{{ nc_probe.rc == 0 }}"
+
+    # ----------------------- DEV1 banner -----------------------
+    - name: Dev1 | Probe banner
+      delegate_to: localhost
+      shell: |
+        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
+        "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
+        "cat /etc/banner | grep -i rev | head -n1"
+      register: dev1_banner
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
+      delegate_to: localhost
+      shell: |
+        printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
+        | awk -F '|' '{print $1}' \
+        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
+        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
+      register: dev1_fw_clean_cmd
+      changed_when: false
+
+    - name: Dev1 | Set final fw string
+      delegate_to: localhost
+      set_fact:
+        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
+
+    - name: Debug Dev1 normalized firmware
+      delegate_to: localhost
+      debug:
+        msg: "Dev1 fw_version  {{ dev1_fw_clean | default('N/A') }}"
+
+    - name: Publish Dev1 fw_version
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'fw_version',
+            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    # ----------------------- DEV2 auth -----------------------
+    - name: Try DEV2 login
+      when: tunnel_ok | default(false)
+      delegate_to: localhost
+      shell: |
+        for f in basicpass basicpass2; do
+          PORT="{{ _local_port }}"
+          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
+            echo "$f"; exit 0
+          fi
+        done
+        exit 1
+      register: dev2_auth
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set dev2_passfile_used
+      delegate_to: localhost
+      set_fact:
+        dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
+
+    # ----------------------- AIRPING #3 -----------------------
+    - name: Refresh ARP #3
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    # ----------------------- DEV2 firmux (simplified, literal) -----------------------
+    - name: Dev2 | Read /usr/lib/release/firmux
+      when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
+      delegate_to: localhost
+      shell: |
+        PORT="{{ _local_port }}"
+        sshpass -f "{{ dev2_passfile_used }}" ssh \
+          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+          -o ConnectTimeout=15 \
+          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+          "cat /usr/lib/release/firmux 2>/dev/null || true"
+      register: dev2_firmux
+      failed_when: false
+      ignore_errors: true
+
+    - name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
+      delegate_to: localhost
+      run_once: true
+      set_fact:
+        indoor_fw_norm: >-
+          {{
+            (
+              (
+                dev2_firmux.stdout | default('') | regex_replace('\r','')
+              ) | split('\n')
+              | map('trim')
+              | select('truthy')
+              | list
+            ) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
+              | list
+              | first
+              | default(
+                  ((dev2_firmux.stdout | default('') | regex_replace('\r',''))
+                  | split('\n')
+                  | map('trim')
+                  | select('truthy')
+                  | list
+                  | first
+                  | default(''))
+                )
+              | trim
+          }}
+
+    - name: Publish Dev2 indoor_fwver
+      delegate_to: localhost
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'custom_field_set',
+            'task_add1': 'indoor_fwver',
+            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+  post_tasks:
+    - name: "✅ Cleanup start"
+      debug:
+        msg: "Closing tunnel and removing temporary IP"
+
+    - name: Refresh ARP #4
+      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Close SSH ControlMaster
+      delegate_to: localhost
+      shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
+      failed_when: false
+      ignore_errors: true
+
+    - name: Remove tunnel control dir
+      delegate_to: localhost
+      file:
+        path: "{{ _ctrl_dir | default('/tmp/none') }}"
+        state: absent
+      ignore_errors: true
+
+    - name: Remove temporary IP
+      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
+      failed_when: false
+      ignore_errors: true
+
+    # --- Determine success of Dev1 & Dev2 reads (controller-side heuristics) ---
+    - name: Post | Derive success flags for Dev1/Dev2 reads
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        dev1_read_ok: "{{ (dev1_fw_clean | default('') | trim) | length > 0 }}"
+        dev2_read_ok: "{{ (dev2_firmux is defined) and ((dev2_firmux.rc | default(1)) == 0) and ((dev2_firmux.stdout | default('') | trim) | length > 0) }}"
+
+    - name: Post | Debug success flags
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "dev1_read_ok={{ dev1_read_ok | default(false) }}"
+          - "dev2_read_ok={{ dev2_read_ok | default(false) }}"
+          - "nb_upgrade_cmd='{{ (nb_upgrade_cmd | default('')) }}' (len={{ (nb_upgrade_cmd | default('')) | length }})"
+
+    # --- If both reads OK and upgrade_cmd looks valid, publish journal + schedule upgrade ---
+    - name: Post | Build journal payload for planned upgrade
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        upgrade_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: >-
+            device is configured to be upgraded with {{ nb_upgrade_cmd }}. requesting the upgrade
+
+    - name: Post | Publish journal to controls
+      when: upgrade_journal_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ upgrade_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_journal_upgrade
+      changed_when: (rmq_journal_upgrade.json is defined) and (rmq_journal_upgrade.json.routed | default(false) | bool)
+      failed_when: false
+
+    - name: Post | Schedule upgrade task via delayed exchange (10s)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
+        method: POST
+        user: "admin"
+        password: "change_me"
+        force_basic_auth: true
+        status_code: 200
+        return_content: yes
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+            headers: { x-delay: 300000 }
+          routing_key: "deviceconfig"
+          payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': (nb_upgrade_cmd | default('')) } | to_json }}"
+          payload_encoding: "string"
+      register: rmq_schedule_upgrade
+      changed_when: false
+      failed_when: false
+
+    # === NEW: Tag only when scheduling path ran (success gate same as scheduling) ===
+    - name: Post | Build tag payload (auto-indoor-upgrade)
+      when:
+        - (dev1_read_ok | default(false)) and (dev2_read_ok | default(false))
+        - (nb_upgrade_cmd | default('')) | length > 10
+        - rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        tag_add_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_add1: "auto-indoor-upgrade"
+
+    - name: Post | Publish add-tag to controls
+      when: tag_add_payload is defined
+      delegate_to: localhost
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_add_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_add
+      changed_when: (rmq_tag_add.json is defined) and (rmq_tag_add.json.routed | default(false) | bool)
+
+    - name: Post | Log upgrade scheduling response
+      when: rmq_schedule_upgrade is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "Upgrade scheduled to deviceconfig.delayed in 10s"
+          - "HTTP status: {{ rmq_schedule_upgrade.status | default('unknown') }}"
+          - "Parsed JSON: {{ rmq_schedule_upgrade.json | default('<none>') }}"
+          - "Raw content: {{ rmq_schedule_upgrade.content | default('<none>') }}"
+
+    - name: "✅ Completed | Device processed"
+      debug:
+        msg:
+          - "Device: {{ inventory_hostname }}"
+          - "Status: DONE"

files/ansible-playbooks/sot-updater-wrapper.yml

@@ -0,0 +1,134 @@
+# sot-updater-wrapper.yml
+# Single nbplay invocation:
+#   Phase 1: Cloud -> NetBox sync (via sot-updater-iponly.yml in wrapper mode)
+#   Phase 2: Refresh ansible_host from nb_ip, reset_connection
+#   Phase 3: Run full sot-updater-current.yml
+
+- hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+  tasks: []
+
+- import_playbook: sot-updater-iponly.yml
+  vars:
+    sot_wrapper_mode: true
+
+
+- name: Phase 2 | Refresh in-memory target IP after iponly
+  hosts: all
+  gather_facts: no
+  vars:
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+
+  tasks:
+    - name: "Wrapper | Extract nb_ip from nb_onedevice_update.py stdout"
+      vars:
+        _lines: "{{ (nb_preflight.stdout_lines | default([])) | map('regex_replace','\r','') | map('trim') | list }}"
+        _ok_line: "{{ (_lines | select('match','^OK\\s+') | list | last | default('')) }}"
+        _fail_line: "{{ (_lines | select('match','^FAIL\\s+') | list | last | default('')) }}"
+        _ip_raw: "{{ _ok_line | regex_search('(?:^|\\s)ip=([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)(?:\\s|$)', '\\1') | default('') }}"
+        _ip: >-
+          {%- if _ip_raw is string -%}
+            {{ _ip_raw }}
+          {%- elif _ip_raw is sequence and (_ip_raw | length) > 0 -%}
+            {{ _ip_raw[0] }}
+          {%- else -%}
+            ""
+          {%- endif -%}
+      set_fact:
+        wrapper_ok_line: "{{ _ok_line }}"
+        wrapper_fail_line: "{{ _fail_line }}"
+        wrapper_nb_ip: "{{ _ip | trim }}"
+
+
+    - name: "Wrapper | Debug extracted values"
+      ansible.builtin.debug:
+        msg:
+          - "wrapper_ok_line='{{ wrapper_ok_line }}'"
+          - "wrapper_fail_line='{{ wrapper_fail_line | default('') }}'"
+          - "wrapper_nb_ip='{{ wrapper_nb_ip }}'"
+
+    # ----------------------- Option A: Cloud offline / no IP -> Journal + stop cleanly -----------------------
+    - name: "Wrapper | Journal: device not online in cloud (skip full scan)"
+      delegate_to: localhost
+      when: (wrapper_fail_line | default('') | length) > 0
+      vars:
+        _journal_msg: >-
+          during sot-updater scan the device was not online in cloud (no IP). please try later.
+          details: {{ wrapper_fail_line | default('') }}
+      uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers: { content-type: "application/json" }
+        body_format: json
+        body:
+          properties: { content_type: "application/json" }
+          routing_key: "{{ control_queue }}"
+          payload: "{{ {
+            'inscope_device': (ansible_hostname | default(inventory_hostname)),
+            'task_name': 'journal_add',
+            'task_result': _journal_msg
+          } | to_json }}"
+          payload_encoding: "string"
+      changed_when: false
+
+    - name: "Wrapper | Mark host to skip full scan (cloud-offline)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      set_fact:
+        wrapper_skip_full_scan: true
+
+
+    - name: "Wrapper | Stop host after cloud-offline journal (no full scan)"
+      when: (wrapper_fail_line | default('') | length) > 0
+      meta: end_host
+
+    # ----------------------- Normal path: we have OK ip=... -> proceed -----------------------
+    - name: "Wrapper | Assert IP extracted successfully"
+      ansible.builtin.assert:
+        that:
+          - nb_script is defined
+          - nb_script.stat.exists | default(false)
+          - (wrapper_ok_line | length) > 0
+          - (wrapper_nb_ip | length) > 0
+        fail_msg: >-
+          Wrapper stop: could not extract IP from nb_onedevice_update.py output.
+          ok_line='{{ wrapper_ok_line | default('') }}'
+          fail_line='{{ wrapper_fail_line | default('') }}'
+          stdout_tail={{ (nb_preflight.stdout_lines | default([]))[-12:] }}
+
+    - name: "Wrapper | Override ansible_host to refreshed IP ({{ wrapper_nb_ip }})"
+      set_fact:
+        ansible_host: "{{ wrapper_nb_ip }}"
+
+    - name: "Wrapper | Reset SSH connection context (ensure new ansible_host is used)"
+      meta: reset_connection
+
+- name: Phase 3 gate | Skip full scan if cloud-offline
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: "Wrapper | Gate: end_host if wrapper_skip_full_scan is set"
+      when: wrapper_skip_full_scan | default(false)
+      meta: end_host
+
+
+- import_playbook: sot-updater.yml

files/ansible-playbooks/update-indoor.yml

@@ -1,4 +1,3 @@
----
 # update-indoor.yml  (conservative, minimal fixes)
 - name: Second-line indoor upgrade via DEV1 → tunnel → DEV2 (non-invasive control path)
   hosts: all
@@ -70,6 +69,28 @@
           - "afterupgrade_routing_key={{ afterupgrade_routing_key | default('UNSET') }}"
           - "inventory_hostname={{ inventory_hostname }}"
 
+    # ====== NEW: pick up force-upgrade flag from CLI and TEMP override ======
+    - name: Read force-upgrade flag from CLI (supports -e force-upgrade=yes)
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "{{ (vars['force-upgrade'] if ('force-upgrade' in vars) else (force_upgrade | default(''))) | string | trim }}"
+
+    # >>> TEMPORARY OVERRIDE (REMOVE THIS TASK LATER) <<<
+    - name: TEMPORARILY set force-upgrade to yes (REMOVE BEFORE COMMITTING)
+      ansible.builtin.set_fact:
+        force_upgrade_raw: "no"
+    # <<< END TEMPORARY >>>
+
+    - name: Normalize force-upgrade flag to boolean
+      ansible.builtin.set_fact:
+        force_upgrade: "{{ (force_upgrade_raw | string | trim) | bool }}"
+
+    - name: Debug force-upgrade effective
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "force_upgrade_raw={{ force_upgrade_raw | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+
     - name: Stop early if connected DEV1 hostname != inventory
       ansible.builtin.meta: end_host
       when: (dev1_host_read.stdout | trim | length > 0) and
@@ -618,6 +639,126 @@
       ansible.builtin.debug:
         msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
 
+    # ====== INSERTED: two-step normalization preview (rev->r, then space->dash) ======
+    - name: "Normalize DEV2 firmux (step 1): replace 'rev ' -> 'r' (debug only)"
+      when: dev2_firmux is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _firmux_step1: "{{ (dev2_firmux.stdout | default('') | trim) | regex_replace('(?i)rev\\s+','r') }}"
+      changed_when: false
+
+    - name: "Debug firmux after step 1 (rev->r)"
+      when: _firmux_step1 is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "_firmux_step1={{ _firmux_step1 }}"
+
+    - name: "Normalize DEV2 firmux (step 2): replace space -> '-' (debug only)"
+      when: _firmux_step1 is defined
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        _firmux_step2: "{{ _firmux_step1 | replace(' ', '-') }}"
+      changed_when: false
+
+    - name: "Debug firmux after step 2 (space->dash)"
+      when: _firmux_step2 is defined
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg: "_firmux_step2={{ _firmux_step2 }}"
+    # ====== END INSERTED ======
+
+
+    # ===================== Early equality/substring check (same logic as checker) =====================
+    # 1) expected_norm from image_filename (prefer extracted X.Y.Z-rNNNN)
+    - name: Normalize expected target step one compute base string (from image_filename)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_step1: "{{ (image_filename | default('') | trim) }}"
+      changed_when: false
+
+    - name: Extract version core X dot Y dot Z dash rNNNN from image_filename if present
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core_list: "{{ (image_filename | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
+      changed_when: false
+
+    - name: Choose first extracted core if available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
+      changed_when: false
+
+    - name: Normalize expected target step two prefer extracted core when available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+      changed_when: false
+
+    # 2) banner_raw + banner_norm (convert "rev NNNN" → "-rNNNN" only if needed)
+    - name: Capture firmware banner line from DEV2 (raw)
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_raw: "{{ (dev2_firmux.stdout | default('') | trim) }}"
+      changed_when: false
+
+    - name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        banner_norm: >-
+          {{
+            ((banner_raw | lower) is search('-r[0-9]+$'))
+            | ternary(
+                banner_raw,
+                (banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1'))
+              )
+          }}
+      changed_when: false
+
+    # 3) evaluate (same equality OR substring, case-insensitive)
+    - name: Evaluate version match using normalized equality or substring
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        version_match: >-
+          {{
+            (expected_norm | default('') | length > 0)
+            and (
+              (banner_norm | default('')) == (expected_norm | default(''))
+              or ((banner_norm | default('') | lower) is search((expected_norm | default('') | lower)))
+              or ((expected_norm | default('') | lower) is search((banner_norm | default('') | lower)))
+            )
+          }}
+      changed_when: false
+
+    - name: Debug  version compare snapshot (pre-write)
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm={{ expected_norm | default('') }}"
+          - "banner_raw={{ banner_raw | default('') }}"
+          - "banner_norm={{ banner_norm | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
+          - "version_match={{ version_match | default(false) }}"
+
+    # 4) soft-block when equal (unless forced) + back-fill legacy debug names
+    - name: Soft-block if already on target (skip staging/write unless force-upgrade)
+      when:
+        - not (force_upgrade | default(false) | bool)
+        - version_match | default(false)
+      ansible.builtin.set_fact:
+        _blocked: true
+        _journal: "{{ (_journal | default([])) + [ 'Device already running target image: ' ~ (banner_norm | default('')) ~ ' — skipping write/flip (use force-upgrade=yes to override)' ] }}"
+        expected_fw_core_early: "{{ expected_norm | default('') }}"
+        current_fw_core: "{{ banner_norm | default('') }}"
+      changed_when: false
+
+    # If we are NOT blocked, still fill the legacy names so your later summary prints them
+    - name: Back-fill compare names for summary (no-op if already set)
+      when: not (_blocked | default(false))
+      ansible.builtin.set_fact:
+        expected_fw_core_early: "{{ expected_fw_core_early | default(expected_norm | default('')) }}"
+        current_fw_core: "{{ current_fw_core | default(banner_norm | default('')) }}"
+      changed_when: false
+
     # ---------------------------- Normalize rebootin early (HOURS) ----------------------------
     # Strict: rebootin is integer HOURS only. Always add +20s grace to the schedule.
 
@@ -952,7 +1093,7 @@
       block:
 
         # --- supervised async write with stall handling ---
-        - name: Write image to inactive bank on DEV2 (update -w, supervised async)
+        - name: Launch update -w on DEV2 (async)
           delegate_to: localhost
           vars:
             _write_async_cap: 600     # 10 minutes max runtime (tune if needed)
@@ -1327,13 +1468,16 @@
       when: afterupgrade_payload is defined
       delegate_to: localhost
 
-    # ---------------------------- Final operator summary (concise) ----------------------------
+    # ---------------------------- Final operator summary (one-liners) ----------------------------
     - name: Summary key outcomes (one-liners)
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
           - "dev2_passfile_used={{ dev2_passfile_used }}"
           - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
+          - "expected_fw_core_early={{ expected_fw_core_early | default('') }}"
+          - "current_fw_core={{ current_fw_core | default('') }}"
+          - "force_upgrade={{ force_upgrade | default(false) }}"
           - "local_image_present={{ local_img.stat.exists | default(false) }}"
           - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
           - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"

files/ansible-playbooks/update-reboot.yml

@@ -1,4 +1,4 @@
 ---
 # update-reboot.yml — thin wrapper, no var forwarding. 
 # Delegates entirely to the unified updater.
-- import_playbook: update-rebootin222.yml
+- import_playbook: update-rebootin223.yml

files/ansible-playbooks/update-rebootin223.yml

@@ -0,0 +1,809 @@
+---
+- name: Upgrade firmware safely (no Python on target)
+  hosts: all
+  gather_facts: no
+
+  vars:
+    # RabbitMQ (pull from env if provided)
+    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
+    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
+    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
+    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
+    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
+    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
+    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
+
+    # NEW: Post-upgrade check scheduling (via holding queue -> DLX)
+    # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'.
+    # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms).
+    # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key,
+    # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side).
+    afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}"
+    afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
+    # Queue name is not used for publish; present for documentation/reference only
+    afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}"
+
+    # REQUIRED (pass via -e)
+    firmware_path: /tmp/2.2.3-r9800.bin
+    firmware_sha256: "5c06496a896831c5548988ac2575a0a84bb9054a769159dec354e1f996c252e4"
+
+    # Helper computed vars
+    fw_base: "{{ firmware_path | basename }}"
+    fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}"
+    fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}"
+    fw_marker: "/tmp/prepared_for_{{ fw_name }}"
+    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
+
+  tasks:
+
+    # ----------------------------- HOSTNAME PREFLIGHT -----------------------------
+    - name: Hostname preflight
+      block:
+        - name: Read remote HOSTNAME
+          ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\""
+          register: host_env
+          changed_when: false
+
+        - name: Debug hostnames
+          ansible.builtin.debug:
+            msg:
+              - "remote_hostname={{ host_env.stdout | trim }}"
+              - "inventory_hostname={{ inventory_hostname }}"
+
+        - name: Stop if connected hostname differs from inventory
+          ansible.builtin.fail:
+            msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'."
+          when: (host_env.stdout | trim) != inventory_hostname
+
+      rescue:
+        - name: Build failure task name and detail (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars (hostname preflight)
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal (hostname preflight)
+          ansible.builtin.set_fact:
+            journal_failure_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_pre_resp
+          changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_pre_resp.status != 200) or
+            (rmq_journal_pre_resp.json is not defined) or
+            (not (rmq_journal_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag (hostname preflight)
+          ansible.builtin.set_fact:
+            tag_failed_payload_pre:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue (hostname preflight)
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload_pre | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_pre_resp
+          changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_pre_resp.status != 200) or
+            (rmq_tag_failed_pre_resp.json is not defined) or
+            (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Stop play after hostname preflight failure
+          ansible.builtin.meta: end_play
+
+    # --- Tag device as update-in-progress at start ---
+    - name: Build control queue payload for update-in-progress tag
+      ansible.builtin.set_fact:
+        tag_inprogress_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "tag_add"
+          task_result: "update-in-progress"
+
+    - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ tag_inprogress_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_tag_inprogress_resp
+      changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_tag_inprogress_resp.status != 200) or
+        (rmq_tag_inprogress_resp.json is not defined) or
+        (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+
+    - name: Log control queue tag publish result
+      ansible.builtin.debug:
+        var: rmq_tag_inprogress_resp.json
+      when: rmq_tag_inprogress_resp is defined
+
+    # --------------------- Prepared marker check BEFORE SSID scan -----------------
+    - name: Check if any prepared marker exists
+      ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT"
+      register: prep_scan
+      changed_when: false
+
+    - name: Debug marker presence
+      ansible.builtin.debug:
+        msg: "prepared_marker={{ prep_scan.stdout | trim }}"
+
+    - name: Journal preparation already present, skipping update steps
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: >-
+            {{
+              {
+                "inscope_device": (ansible_hostname | default(inventory_hostname)),
+                "task_name": "journal_add",
+                "task_result": (
+                  "Preparation already present for " ~ fw_banner_repr ~
+                  "; marker " ~ fw_marker ~
+                  ". Skipping update steps."
+                )
+              } | to_json
+            }}
+          payload_encoding: "string"
+      register: rmq_journal_prep_present
+      changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_journal_prep_present.status != 200) or
+        (rmq_journal_prep_present.json is not defined) or
+        (not (rmq_journal_prep_present.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+
+    - name: End play for this host (already prepared)
+      ansible.builtin.meta: end_host
+      when: (prep_scan.stdout | trim) == 'PRESENT'
+
+    # --- SSID scan & journal (does not stop the play) ---
+    - name: Count SSID lines in /tmp/config.json (filtered)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l
+      register: ssid_lines
+      changed_when: false
+
+    - name: Debug SSID count
+
+      ansible.builtin.debug:
+        msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}"
+
+    - name: Build joined SSID list when multiple SSIDs found (≥3)
+      ansible.builtin.raw: >
+        {{ pathprefix }}
+        grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \
+        | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \
+        | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }'
+      register: ssid_concat
+      changed_when: false
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Build control queue payload for SSID journal (journal_add)
+      ansible.builtin.set_fact:
+        ssid_journal_payload:
+          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+          task_name: "journal_add"
+          task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}"
+      when: (ssid_lines.stdout | trim | int) >= 3
+
+    - name: Publish SSID journal to control queue via RabbitMQ HTTP API
+      ansible.builtin.uri:
+        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+        method: POST
+        user: "{{ rmq_user }}"
+        password: "{{ rmq_pass }}"
+        force_basic_auth: true
+        status_code: 200
+        headers:
+          content-type: "application/json"
+        body_format: json
+        body:
+          properties:
+            content_type: "application/json"
+          routing_key: "{{ control_queue }}"
+          payload: "{{ ssid_journal_payload | to_json }}"
+          payload_encoding: "string"
+      register: rmq_ssid_journal_resp
+      changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool)
+      failed_when: >
+        (rmq_ssid_journal_resp.status != 200) or
+        (rmq_ssid_journal_resp.json is not defined) or
+        (not (rmq_ssid_journal_resp.json.routed | default(false) | bool))
+      delegate_to: localhost
+      when: ssid_journal_payload is defined
+
+    # ----------------------------- MAIN UPDATE BLOCK -----------------------------
+    - name: Firmware update main block
+      block:
+
+        - name: Check if firmware image is already on the device
+          ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING"
+          register: fw_exists
+          changed_when: false
+
+        - name: Count fw_printenv lines
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l"
+          register: env_line_count
+          changed_when: false
+
+        - name: Debug fw_printenv line count
+          ansible.builtin.debug:
+            msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}"
+
+        - name: Stop if bootloader environment looks invalid (<7 lines)
+          ansible.builtin.fail:
+            msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted."
+          when: (env_line_count.stdout | trim | int) < 7
+
+        - name: Read first line of /etc/banner (current running version)
+          ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1"
+          register: banner
+          changed_when: false
+
+        - name: current version
+          ansible.builtin.debug:
+            msg:
+              - "current banner: {{ banner.stdout | trim }}"
+
+        - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }})
+          ansible.builtin.fail:
+            msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})"
+          when: banner.stdout is search(fw_banner_repr)
+
+        - name: Upload firmware to /tmp via scp (controller-side)
+          ansible.builtin.shell: |
+            set -e
+            SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.3-r9800.bin") }}'
+            DST_USER='{{ ansible_user | default("root") }}'
+            DST_HOST='{{ ansible_host | default(inventory_hostname) }}'
+            test -f "$SRC"
+            sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+              "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}"
+          delegate_to: localhost
+          when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK'
+          changed_when: true
+
+        - name: Re-check firmware presence after optional upload
+          ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING"
+          register: fw_exists2
+          changed_when: false
+          failed_when: (fw_exists2.stdout | trim) != 'OK'
+
+        - name: Compute sha256 of the uploaded image
+          ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'"
+          register: sha_out
+          changed_when: false
+
+        - name: Verify sha256 matches expected
+          ansible.builtin.fail:
+            msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}"
+          when: (sha_out.stdout | trim) != (firmware_sha256 | trim)
+
+        - name: sha256 verification debug
+          ansible.builtin.debug:
+            msg:
+              - "sha256sum is: {{ sha_out.stdout | trim }}"
+
+        - name: Check image validity (update -c must say 'valid')
+          ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'"
+          register: up_check
+          changed_when: false
+          failed_when: up_check.stdout.strip() != 'valid'
+
+        - name: image verification debug
+          ansible.builtin.debug:
+            msg:
+              - ".bin verification is: {{ up_check.stdout | trim }}"
+
+        #    - name: forced stop before writing
+        #      ansible.builtin.meta: end_play
+
+        - name: Write image (this will take a while)
+          ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'"
+          register: up_write
+          changed_when: true
+          failed_when: up_write.stdout is not search('update is complete')
+
+        - name: Read current active partition
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_before
+          changed_when: false
+          failed_when: active_before.stdout | trim not in ['1','2']
+
+        - name: Determine new active value
+          ansible.builtin.set_fact:
+            new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}"
+
+        - name: Switch active partition to {{ new_active }}
+          ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}"
+          register: setenv_out
+          changed_when: true
+
+        - name: Verify active partition flipped
+          ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'"
+          register: active_after
+          changed_when: false
+          failed_when: (active_after.stdout | trim) != new_active
+
+        - name: Create prepared marker
+          ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'"
+          changed_when: true
+
+        - name: Build control queue payload (progress & target version)
+          ansible.builtin.set_fact:
+            nbq2_payload_obj:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "update_wo_restart"
+              task_result: "waiting_restart"
+              task_add1: "{{ fw_banner_repr }}"     # e.g., "2.2.0 rev 9739"
+          when: up_write is changed
+
+        - name: Publish message to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ nbq2_payload_obj | to_json }}"
+              payload_encoding: "string"
+          register: rmq_resp
+          changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false))
+          failed_when: >
+            (rmq_resp.status != 200) or
+            (rmq_resp.json is not defined) or
+            (rmq_resp.json.routed | default(false) | bool == false)
+          delegate_to: localhost
+          when: nbq2_payload_obj is defined
+
+        - name: Log control queue publish result
+          ansible.builtin.debug:
+            var: rmq_resp.json
+          when: rmq_resp is defined
+
+        - name: Summary
+          ansible.builtin.debug:
+            msg:
+              - "Banner before: {{ banner.stdout | trim }}"
+              - "Target version: {{ fw_banner_repr }}"
+              - "SHA256: OK ({{ sha_out.stdout | trim }})"
+              - "update -c: {{ up_check.stdout | trim }}"
+              - "update -w: OK"
+              - "active: {{ active_before.stdout | trim }} -> {{ new_active }}"
+              - "Marker: {{ fw_marker }}"
+
+        # --- Optional scheduled reboot (must be last device-side command) ---
+        - name: Compute reboot delay in seconds (if rebootin provided)
+          ansible.builtin.set_fact:
+            reboot_seconds: "{{ (rebootin | int) * 3600 }}"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+
+        - name: Schedule delayed reboot on device (HUP-safe)
+          ansible.builtin.raw: >
+            {{ pathprefix }}
+            sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &'
+          changed_when: true
+          when:
+            - nbq2_payload_obj is defined
+            - reboot_seconds is defined
+
+        # --- Success tag selection (ONLY CHANGE) ---
+        - name: Build control queue payload for update-auto-restarted (rebootin == 0)
+          ansible.builtin.set_fact:
+            tag_auto_restarted_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-auto-restarted"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) == 0
+
+        - name: Build control queue payload for update-restart-scheduled (rebootin >= 1)
+          ansible.builtin.set_fact:
+            tag_restart_scheduled_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-restart-scheduled"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is defined
+            - (rebootin | int) >= 1
+
+        - name: Build control queue payload for update-waits-restart tag (no reboot scheduled)
+          ansible.builtin.set_fact:
+            tag_waits_restart_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-waits-restart"
+          when:
+            - nbq2_payload_obj is defined
+            - rebootin is not defined
+
+        # --- Publish chosen tag (updated names only) ---
+        - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_waits_restart_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_waits_restart_resp
+          changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_waits_restart_resp.status != 200) or
+            (rmq_tag_waits_restart_resp.json is not defined) or
+            (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_waits_restart_payload is defined
+
+        - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_auto_restarted_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_auto_restarted_resp
+          changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_auto_restarted_resp.status != 200) or
+            (rmq_tag_auto_restarted_resp.json is not defined) or
+            (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_auto_restarted_payload is defined
+
+        - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_restart_scheduled_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_restart_scheduled_resp
+          changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_restart_scheduled_resp.status != 200) or
+            (rmq_tag_restart_scheduled_resp.json is not defined) or
+            (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+          when: tag_restart_scheduled_payload is defined
+
+        # --- Journal: preparation successful (only if fully successful) ---
+        - name: Build control queue payload for success journal
+          ansible.builtin.set_fact:
+            journal_success_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: >-
+                Preparation complete for {{ fw_banner_repr }}.
+                Active {{ active_before.stdout | trim }} → {{ new_active }};
+                marker {{ fw_marker }}.
+                {{
+                  ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.')
+                  if (rebootin is defined)
+                  else 'Waiting for restart to activate new firmware.'
+                }}
+          when: nbq2_payload_obj is defined
+
+        - name: Publish success journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_success_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_success_resp
+          changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_success_resp.status != 200) or
+            (rmq_journal_success_resp.json is not defined) or
+            (not (rmq_journal_success_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        # ----------------- NEW: schedule "afterupgrade_check" message -------------
+        # Architectural notes:
+        # - Only schedule if preparation succeeded (nbq2_payload_obj set)
+        # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer
+        #   by re-enqueuing fresh messages with increased delays; the producer does NOT sleep.
+        # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms).
+        #   After TTL, the holding queue dead-letters to exchange 'deviceconfig'.
+        - name: Init after-upgrade scheduling vars
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_max_attempts: 3
+            # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m).
+            # Because this task runs with delegate_to: localhost, read from hostvars.
+            au_delay_sec: >-
+              {{
+                (
+                  (hostvars[inventory_hostname].reboot_seconds | default(0) | int)
+                  + 300
+                )
+                if (hostvars[inventory_hostname].reboot_seconds is defined)
+                else 300
+              }}
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        # NEW: compute values that the payload will reference
+        - name: Generate correlation ID and original emitted timestamp
+          ansible.builtin.set_fact:
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+
+        - name: Build after-upgrade check payload (attempt 1)
+          ansible.builtin.set_fact:
+            au_attempt: 1
+            au_delay_sec: "{{ au_delay_sec | default(300) }}"
+            au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}"
+            au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}"
+            afterupgrade_payload:
+              task_name: "afterupgrade_check"
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              target_version: "{{ fw_banner_repr }}"
+              attempt: "{{ au_attempt | default(1) }}"
+              max_attempts: "{{ au_max_attempts | default(3) }}"
+              current_delay_sec: "{{ au_delay_sec | default(300) }}"
+              correlation_id: "{{ au_correlation_id }}"
+              original_emitted_at: "{{ au_original_emitted_at }}"
+              schema_version: 1
+          when: nbq2_payload_obj is defined
+          delegate_to: localhost
+
+        - name: Debug x-delay about to be sent (ms)
+          ansible.builtin.debug:
+            msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}"
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+        - name: DEBUG after-upgrade payload and timing
+          delegate_to: localhost
+          ansible.builtin.debug:
+            msg:
+              - "afterupgrade_payload={{ afterupgrade_payload | to_nice_json }}"
+              - "au_delay_sec={{ au_delay_sec }}"
+              - "reboot_seconds(host)={{ hostvars[inventory_hostname].reboot_seconds | default('undefined') }}"
+
+
+        - name: Publish delayed after-upgrade check to holding exchange
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+                headers:
+                  x-delay: "{{ (au_delay_sec | int) * 1000 }}"
+              routing_key: "{{ afterupgrade_routing_key }}"
+              payload: "{{ afterupgrade_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_afterupgrade_resp
+          changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_afterupgrade_resp.status != 200) or
+            (rmq_afterupgrade_resp.json is not defined)
+          when: afterupgrade_payload is defined
+          delegate_to: localhost
+
+      rescue:
+        - name: Build failure task name and detail
+          ansible.builtin.set_fact:
+            fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}"
+            fail_detail_raw: >-
+              {{ ansible_failed_result.msg
+                 | default(ansible_failed_result.stderr)
+                 | default(ansible_failed_result.stdout)
+                 | default('no additional error output')
+                 | trim }}
+
+        - name: Build failure summary text
+          ansible.builtin.set_fact:
+            fail_summary: >-
+              Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }}
+
+        - name: Truncate failure summary to ~400 chars
+          ansible.builtin.set_fact:
+            fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}"
+
+        - name: Build control queue payload for failure journal
+          ansible.builtin.set_fact:
+            journal_failure_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "journal_add"
+              task_result: "{{ fail_summary_short }}"
+
+        - name: Publish failure journal to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ journal_failure_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_journal_fail_resp
+          changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_journal_fail_resp.status != 200) or
+            (rmq_journal_fail_resp.json is not defined) or
+            (not (rmq_journal_fail_resp.json.routed | default(false) | bool))
+          delegate_to: localhost
+
+        - name: Build control queue payload for update-aborted tag
+          ansible.builtin.set_fact:
+            tag_failed_payload:
+              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+              task_name: "tag_add"
+              task_result: "update-aborted"
+
+        - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API
+          ansible.builtin.uri:
+            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+            method: POST
+            user: "{{ rmq_user }}"
+            password: "{{ rmq_pass }}"
+            force_basic_auth: true
+            status_code: 200
+            headers:
+              content-type: "application/json"
+            body_format: json
+            body:
+              properties:
+                content_type: "application/json"
+              routing_key: "{{ control_queue }}"
+              payload: "{{ tag_failed_payload | to_json }}"
+              payload_encoding: "string"
+          register: rmq_tag_failed_resp
+          changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool)
+          failed_when: >
+            (rmq_tag_failed_resp.status != 200) or
+            (rmq_tag_failed_resp.json is not defined) or
+            (not (rmq_tag_failed_resp.json.routed | default(false) | bool))
+          delegate_to: localhost

files/nb_onedevice_update.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""
+nb_sync_one_device.py
+
+Update a single NetBox device from Cloud by hostname — ONLY if device is online.
+
+Behavior
+- Liveness gate via /v1/devices/{cloud_id}/liveness (no changes if offline).
+- Updates NetBox fields from Cloud:
+  custom_fields.fw_version ← firmwareVersion
+  custom_fields.nodeName   ← nodeName
+  custom_fields.sectorName ← sectorName
+  custom_fields.smallCellName ← smallCellName
+  serial ← serialNumber
+- IP handling:
+  If Cloud ipAddress is valid (not None/""/"0.0.0.0"):
+    ensure eth0, ensure/create IP, MOVE from other device if needed, set primary_ip4,
+    then PRUNE all other IPs on this device (default).
+  If Cloud ipAddress is placeholder/invalid:
+    skip IP changes and do not prune.
+- --chatty logs step-by-step to stderr; stdout remains one-line OK/FAIL.
+- NEW: Logs custom field upgrade_cmd as `NB: upgrade_cmd=<value>` when --chatty.
+
+Exit codes:
+  0 = success
+  1 = not found / missing data / offline
+  3 = network/HTTP error
+  4 = NetBox update error
+"""
+
+import sys
+import json
+import argparse
+from typing import Optional, Union, List, Dict
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# ------------ HARD-CODED CONFIG (per Pavel) ------------
+NB_URL = "http://netbox.gt-tiso.ikeja.co.za"
+NB_TOKEN = "7648e4f5ee370cda7834682e61b47c2ee8e95623"
+CLOUD_API_BASE = "https://cloud.ikeja.co.za/v1/devices"
+CLOUD_BEARER = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzY2OTk3ODQ2LCJleHAiOjE3Njk1ODk4NDZ9.aBuEOQOC6i5jX6Ixwb3gabzV4uIeYJdbpJodxQy1DkE"
+
+REQ_TIMEOUT = 30
+CHATTY = False
+
+# ------------ HTTP utilities ------------
+def _new_session() -> requests.Session:
+    s = requests.Session()
+    retries = Retry(
+        total=3, connect=3, read=3, status=3,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=("GET", "POST", "PATCH", "PUT", "DELETE"),
+        respect_retry_after_header=True,
+    )
+    adapter = HTTPAdapter(max_retries=retries, pool_connections=8, pool_maxsize=16)
+    s.mount('http://', adapter)
+    s.mount('https://', adapter)
+    s.headers['Accept'] = 'application/json'
+    return s
+
+S_NB = _new_session()
+S_CL = _new_session()
+
+# ------------ Logging / status helpers ------------
+def log(msg: str):
+    if CHATTY:
+        print(msg, file=sys.stderr)
+
+def die(code: int, msg: str):
+    # Single-line result: stdout on success, stderr on failure
+    print(msg, file=sys.stdout if code == 0 else sys.stderr)
+    raise SystemExit(code)
+
+# ------------ NetBox API ------------
+class NetBox:
+    def __init__(self, base: str, token: str):
+        self.base = base.rstrip('/')
+        self.token = token
+
+    def _h(self):
+        return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
+
+    def _url(self, path: str) -> str:
+        return f"{self.base}{path}"
+
+    def get_device_by_name(self, name: str) -> Optional[dict]:
+        log(f"NB: lookup device by name {name}")
+        r = S_NB.get(self._url("/api/dcim/devices/"), headers=self._h(),
+                     params={"name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET devices name={name} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def get_device(self, dev_id: int) -> dict:
+        log(f"NB: fetch device id={dev_id}")
+        r = S_NB.get(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET device id={dev_id} HTTP={r.status_code}")
+        return r.json()
+
+    def patch_device(self, dev_id: int, patch: dict) -> None:
+        if not patch:
+            log("NB: no device patch needed")
+            return
+        log(f"NB: patch device id={dev_id} keys={list(patch.keys())}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps(patch), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL patch device dev={dev_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_iface_id(self, dev_id: int, name: str) -> Optional[int]:
+        r = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                     params={"device_id": dev_id, "name": name}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET interfaces device_id={dev_id} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0]["id"] if res else None
+
+    def ensure_eth0(self, dev_id: int) -> int:
+        ifid = self.get_iface_id(dev_id, "eth0")
+        log(f"NB: ensure eth0 (current id={ifid})")
+        if ifid:
+            return ifid
+        payload = {"device": dev_id, "name": "eth0", "type": "1000base-t"}
+        r = S_NB.post(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        if r.status_code == 400:
+            # race: read again
+            ifid = self.get_iface_id(dev_id, "eth0")
+            if ifid:
+                return ifid
+        die(4, f"FAIL create eth0 HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_ip_by_addr(self, addr: str) -> Optional[dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"address": f"{addr}/32"}, timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL NetBox GET ip-addresses addr={addr} HTTP={r.status_code}")
+        res = r.json().get("results") or []
+        return res[0] if res else None
+
+    def create_ip_for_iface(self, addr: str, iface_id: int) -> int:
+        payload = {
+            "address": f"{addr}/32",
+            "status": "active",
+            "assigned_object_type": "dcim.interface",
+            "assigned_object_id": iface_id,
+        }
+        r = S_NB.post(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                      data=json.dumps(payload), timeout=REQ_TIMEOUT)
+        if r.status_code == 201:
+            return r.json()["id"]
+        die(4, f"FAIL create IP {addr} HTTP={r.status_code} body={r.text[:200]}")
+
+    def assign_ip_to_iface(self, ip_id: int, iface_id: int) -> bool:
+        log(f"NB: assign IP id={ip_id} -> iface={iface_id}")
+        r = S_NB.patch(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(),
+                       data=json.dumps({
+                           "assigned_object_type": "dcim.interface",
+                           "assigned_object_id": iface_id
+                       }),
+                       timeout=REQ_TIMEOUT)
+        return 200 <= r.status_code < 300
+
+    def device_set_primary_ip4(self, dev_id: int, ip_id: int) -> None:
+        log(f"NB: set primary_ip4 dev={dev_id} -> ip_id={ip_id}")
+        r = S_NB.patch(self._url(f"/api/dcim/devices/{dev_id}/"), headers=self._h(),
+                       data=json.dumps({"primary_ip4": ip_id}), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300):
+            die(4, f"FAIL set primary_ip4 dev={dev_id} ip_id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+    def get_device_id_of_interface(self, iface_id: int) -> Optional[int]:
+        r = S_NB.get(self._url(f"/api/dcim/interfaces/{iface_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return (r.json().get("device") or {}).get("id")
+        return None
+
+    # --- Helpers for pruning ---
+    def list_device_ips(self, dev_id: int) -> List[Dict]:
+        r = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                     params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r.status_code == 200:
+            return r.json().get("results") or []
+        ips: List[Dict] = []
+        r2 = S_NB.get(self._url("/api/dcim/interfaces/"), headers=self._h(),
+                      params={"device_id": dev_id, "limit": 1000}, timeout=REQ_TIMEOUT)
+        if r2.status_code == 200:
+            for iface in (r2.json().get("results") or []):
+                ifid = iface.get("id")
+                r3 = S_NB.get(self._url("/api/ipam/ip-addresses/"), headers=self._h(),
+                              params={"assigned_object_type": "dcim.interface",
+                                      "assigned_object_id": ifid, "limit": 1000}, timeout=REQ_TIMEOUT)
+                if r3.status_code == 200:
+                    ips.extend(r3.json().get("results") or [])
+        return ips
+
+    def delete_ip(self, ip_id: int) -> None:
+        log(f"NB: delete IP id={ip_id}")
+        r = S_NB.delete(self._url(f"/api/ipam/ip-addresses/{ip_id}/"), headers=self._h(), timeout=REQ_TIMEOUT)
+        if not (200 <= r.status_code < 300 or r.status_code == 204):
+            die(4, f"FAIL delete IP id={ip_id} HTTP={r.status_code} body={r.text[:200]}")
+
+# ------------ Cloud API ------------
+class Cloud:
+    def __init__(self, base: str, bearer: str):
+        self.base = base.rstrip("/")
+        self.bearer = bearer
+
+    def _h(self):
+        return {"Authorization": f"Bearer {self.bearer}", "Accept": "application/json"}
+
+    def device_detail(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}"
+        log(f"CL: fetch detail cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id} HTTP={r.status_code}")
+        return r.json()
+
+    def device_liveness(self, cloud_id: Union[str, int]) -> dict:
+        url = f"{self.base}/{cloud_id}/liveness"
+        log(f"CL: fetch liveness cloud_id={cloud_id}")
+        r = S_CL.get(url, headers=self._h(), timeout=REQ_TIMEOUT)
+        if r.status_code != 200:
+            die(3, f"FAIL Cloud GET {cloud_id}/liveness HTTP={r.status_code}")
+        return r.json()
+
+# ------------ Core ------------
+def _primary_ip4_text(dev_json: dict) -> Optional[str]:
+    p = dev_json.get("primary_ip4") or {}
+    addr = p.get("address")
+    if isinstance(addr, str) and addr.endswith("/32"):
+        return addr[:-3]
+    return addr
+
+def run(hostname: str) -> None:
+    nb = NetBox(NB_URL, NB_TOKEN)
+    cl = Cloud(CLOUD_API_BASE, CLOUD_BEARER)
+
+    dev = nb.get_device_by_name(hostname)
+    if not dev:
+        die(1, f"FAIL {hostname} not found in NetBox")
+
+    dev_id = dev["id"]
+    dev_full = nb.get_device(dev_id)
+    cf = dev_full.get("custom_fields") or {}
+
+    # NEW: log upgrade command if present
+    upgrade_cmd = cf.get("upgrade_cmd")
+    log(f"NB: upgrade_cmd={upgrade_cmd}")
+
+    cloud_id = cf.get("cloud_id")
+    if cloud_id in (None, "", "null"):
+        die(1, f"FAIL {hostname} has no custom_fields.cloud_id in NetBox")
+
+    # 1) Liveness gate
+    live = cl.device_liveness(cloud_id)
+    if not bool(live.get("isConnected")):
+        die(1, f"FAIL {hostname} cloud_id={cloud_id} device is not online")
+
+    current_nb_ip = _primary_ip4_text(dev_full)
+
+    # 2) Cloud detail
+    d = cl.device_detail(cloud_id)
+    fw = d.get("firmwareVersion") or d.get("version")
+    ip_from_cloud = (d.get("ipAddress") or "").strip() if isinstance(d.get("ipAddress"), str) else d.get("ipAddress")
+    node = d.get("nodeName")
+    sector = d.get("sectorName")
+    small = d.get("smallCellName")
+    serial = d.get("serialNumber")
+
+    if not fw:
+        die(1, f"FAIL {hostname} cloud_id={cloud_id}: missing firmwareVersion")
+
+    # 3) NetBox patch (idempotent)
+    cf_patch = {}
+    if cf.get("fw_version") != fw:
+        cf_patch["fw_version"] = fw
+        log(f"CF: fw_version -> {fw}")
+    if node and cf.get("nodeName") != node:
+        cf_patch["nodeName"] = node
+        log(f"CF: nodeName -> {node}")
+    if sector and cf.get("sectorName") != sector:
+        cf_patch["sectorName"] = sector
+        log(f"CF: sectorName -> {sector}")
+    if small and cf.get("smallCellName") != small:
+        cf_patch["smallCellName"] = small
+        log(f"CF: smallCellName -> {small}")
+
+    dev_patch = {}
+    if serial and (dev_full.get("serial") != serial):
+        dev_patch["serial"] = serial
+        log(f"DEV: serial -> {serial}")
+    if cf_patch:
+        dev_patch["custom_fields"] = cf_patch
+    nb.patch_device(dev_id, dev_patch)
+
+    # 4) IP handling (skip if placeholder)
+    ip_is_placeholder = (ip_from_cloud in (None, "", "0.0.0.0"))
+    ip_out_for_status = current_nb_ip  # default to current NB IP
+
+    if ip_is_placeholder:
+        log(f"IP: cloud reported placeholder '{ip_from_cloud}', skipping IP changes; keeping NetBox ip(s) as-is")
+    else:
+        iface_id = nb.ensure_eth0(dev_id)
+
+        ip_rec = nb.get_ip_by_addr(ip_from_cloud)
+        if ip_rec is None:
+            log(f"IP: create new {ip_from_cloud} on iface {iface_id}")
+            ip_id = nb.create_ip_for_iface(ip_from_cloud, iface_id)
+        else:
+            ip_id = ip_rec["id"]
+            assigned_type = ip_rec.get("assigned_object_type") or ""
+            assigned_id = ip_rec.get("assigned_object_id")
+            if not assigned_type:
+                if not nb.assign_ip_to_iface(ip_id, iface_id):
+                    die(4, f"FAIL assign IP {ip_from_cloud} to iface {iface_id}")
+            elif assigned_type == "dcim.interface":
+                if str(assigned_id) != str(iface_id):
+                    other_dev = nb.get_device_id_of_interface(assigned_id) if assigned_id else None
+                    log(f"IP: moving {ip_from_cloud} from iface={assigned_id} dev={other_dev} -> iface={iface_id} dev={dev_id}")
+                    # Clear old device primary if necessary
+                    if other_dev:
+                        r = S_NB.get(nb._url(f"/api/dcim/devices/{other_dev}/"), headers=nb._h(), timeout=REQ_TIMEOUT)
+                        if r.status_code == 200:
+                            old_primary_id = (r.json().get("primary_ip4") or {}).get("id")
+                            if str(old_primary_id) == str(ip_id):
+                                S_NB.patch(nb._url(f"/api/dcim/devices/{other_dev}/"),
+                                           headers=nb._h(), data=json.dumps({"primary_ip4": None}),
+                                           timeout=REQ_TIMEOUT)
+                    if not nb.assign_ip_to_iface(ip_id, iface_id):
+                        die(4, f"FAIL move IP {ip_from_cloud} to iface {iface_id}")
+            else:
+                die(4, f"FAIL IP {ip_from_cloud} assigned to {assigned_type}")
+
+        nb.device_set_primary_ip4(dev_id, ip_id)
+        ip_out_for_status = ip_from_cloud
+
+        # PRUNE all other IPs on this device (default behavior)
+        all_ips = nb.list_device_ips(dev_id)
+        for rec in all_ips:
+            rid = rec.get("id")
+            if str(rid) == str(ip_id):
+                continue
+            addr = rec.get("address")
+            log(f"IP: pruning stale {addr} (id={rid}) from device {dev_id}")
+            nb.delete_ip(rid)
+
+    print(f"OK {hostname} ip={ip_out_for_status or 'NONE'} fw={fw} node={node} sector={sector} small={small}")
+
+# ------------ CLI ------------
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description="Sync one NetBox device from Cloud by hostname (only if online)")
+    ap.add_argument("hostname", help="Device name in NetBox")
+    ap.add_argument("--chatty", action="store_true", help="Verbose step-by-step logging to stderr")
+    args = ap.parse_args()
+
+    CHATTY = bool(args.chatty)  # module-scope assignment
+    try:
+        run(args.hostname)
+    except requests.RequestException as e:
+        die(3, f"FAIL network error: {e}")

files/pppoe_to_dhcp.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+Convert a device config from PPPoE uplink to DHCP on VLAN 4000.
+
+Edits ONLY these paths:
+  .network.zones.wan.mode                         -> "dhcp"
+  .network.zones.wan.alias                        -> []          (ensure present)
+  .network.zones.wan.stp                          -> false       (ensure present)
+  .network.zones.wan.dns                          -> ["41.222.55.1"]
+  .ethernet.ports.eth0.network.vlan_access.enabled-> true
+  .ethernet.ports.eth0.network.vlan_access.id     -> 4000
+
+Everything else is preserved as-is (no key sorting, order preserved).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+from collections import OrderedDict
+from typing import Any, MutableMapping, Sequence
+
+
+def load_json_preserve_order(path: str) -> Any:
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f, object_pairs_hook=OrderedDict)
+
+
+def detect_indent(raw_text: str) -> int:
+    """
+    Best-effort indent detection. Defaults to 2 if unclear.
+    """
+    # Look for the first line that begins with spaces then a quote (a key).
+    m = re.search(r"\n( +)\"", raw_text)
+    if not m:
+        return 2
+    spaces = len(m.group(1))
+    # Common indents are 2 or 4; accept any positive count.
+    return spaces if spaces > 0 else 2
+
+
+def get_mapping(root: Any, path: Sequence[str]) -> MutableMapping[str, Any]:
+    """
+    Walks down dict-like objects; raises KeyError/TypeError if structure is missing.
+    Returns the mapping at the end of the path.
+    """
+    cur = root
+    for key in path:
+        if not isinstance(cur, MutableMapping):
+            raise TypeError(f"Expected object at {'.'.join(path)}, got {type(cur).__name__}")
+        if key not in cur:
+            raise KeyError(f"Missing key '{key}' at {'.'.join(path)}")
+        cur = cur[key]
+    if not isinstance(cur, MutableMapping):
+        raise TypeError(f"Expected object at {'.'.join(path)}, got {type(cur).__name__}")
+    return cur
+
+
+def ensure_path(root: Any, path: Sequence[str]) -> MutableMapping[str, Any]:
+    """
+    Ensures nested dicts exist; creates missing dicts as OrderedDict.
+    Returns the mapping at the end of the path.
+    """
+    cur = root
+    for key in path:
+        if not isinstance(cur, MutableMapping):
+            raise TypeError(f"Expected object while creating {'.'.join(path)}, got {type(cur).__name__}")
+        if key not in cur or cur[key] is None:
+            cur[key] = OrderedDict()
+        cur = cur[key]
+    if not isinstance(cur, MutableMapping):
+        raise TypeError(f"Expected object at {'.'.join(path)}, got {type(cur).__name__}")
+    return cur
+
+
+def set_value(root: Any, path: Sequence[str], value: Any, create: bool = False) -> tuple[bool, Any, Any]:
+    """
+    Set value at path. If create=False, path must exist. If create=True, missing
+    objects along the way are created.
+    Returns (changed, old_value, new_value).
+    """
+    if len(path) < 1:
+        raise ValueError("Path must have at least one key")
+    parent_path = path[:-1]
+    leaf = path[-1]
+
+    parent = ensure_path(root, parent_path) if create else get_mapping(root, parent_path)
+
+    old = parent.get(leaf, None)
+    if old == value:
+        return (False, old, value)
+    parent[leaf] = value
+    return (True, old, value)
+
+
+def validate_post(root: Any) -> None:
+    """
+    Minimal structural and value validation for the changed fields.
+    Raises exceptions on mismatch.
+    """
+    # Check mode
+    wan = get_mapping(root, ["network", "zones", "wan"])
+    if wan.get("mode") != "dhcp":
+        raise ValueError("Post-check failed: .network.zones.wan.mode != 'dhcp'")
+
+    # Check VLAN access
+    vlan_access = get_mapping(root, ["ethernet", "ports", "eth0", "network", "vlan_access"])
+    if vlan_access.get("enabled") is not True:
+        raise ValueError("Post-check failed: vlan_access.enabled is not true")
+    if vlan_access.get("id") != 4000:
+        raise ValueError("Post-check failed: vlan_access.id != 4000")
+
+    # Check schema fields
+    if wan.get("alias") != []:
+        raise ValueError("Post-check failed: .network.zones.wan.alias != []")
+    if wan.get("stp") is not False:
+        raise ValueError("Post-check failed: .network.zones.wan.stp != false")
+
+    # DNS
+    if wan.get("dns") != ["41.222.55.1"]:
+        raise ValueError("Post-check failed: .network.zones.wan.dns != ['41.222.55.1']")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Convert config JSON from PPPoE to DHCP on VLAN 4000 (minimal edits).")
+    ap.add_argument("input", help="Input JSON file (e.g., config.json)")
+    ap.add_argument("-o", "--output", help="Output file. If omitted and --in-place not set, prints to stdout.")
+    ap.add_argument("--in-place", action="store_true", help="Modify input file in place (creates .bak backup).")
+    ap.add_argument("--backup-suffix", default=".bak", help="Backup suffix for --in-place (default: .bak)")
+    args = ap.parse_args()
+
+    in_path = args.input
+
+    raw = open(in_path, "r", encoding="utf-8").read()
+    indent = detect_indent(raw)
+
+    data = load_json_preserve_order(in_path)
+
+    changes: list[str] = []
+
+    def apply(path: Sequence[str], value: Any, create: bool = False) -> None:
+        changed, old, new = set_value(data, path, value, create=create)
+        if changed:
+            changes.append(f"{'.' + '.'.join(path)}: {old!r} -> {new!r}")
+
+    # Required edits
+    apply(["network", "zones", "wan", "mode"], "dhcp", create=False)
+
+    # "Only in new" fields must exist exactly like new
+    apply(["network", "zones", "wan", "alias"], [], create=True)
+    apply(["network", "zones", "wan", "stp"], False, create=True)
+
+    # DNS pinned as requested
+    apply(["network", "zones", "wan", "dns"], ["41.222.55.1"], create=True)
+
+    # VLAN 4000 on eth0
+    apply(["ethernet", "ports", "eth0", "network", "vlan_access", "enabled"], True, create=True)
+    apply(["ethernet", "ports", "eth0", "network", "vlan_access", "id"], 4000, create=True)
+
+    # Validate
+    validate_post(data)
+
+    # Serialize
+    out_text = json.dumps(data, indent=indent, ensure_ascii=False) + "\n"
+
+    # Write
+    if args.in_place:
+        backup_path = in_path + args.backup_suffix
+        if not os.path.exists(backup_path):
+            shutil.copy2(in_path, backup_path)
+        else:
+            # Avoid overwriting an existing backup silently
+            raise FileExistsError(f"Backup already exists: {backup_path}")
+        with open(in_path, "w", encoding="utf-8") as f:
+            f.write(out_text)
+    elif args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(out_text)
+    else:
+        sys.stdout.write(out_text)
+
+    # Report to stderr for CLI usage
+    sys.stderr.write("Applied changes:\n")
+    if changes:
+        for line in changes:
+            sys.stderr.write(f"  - {line}\n")
+    else:
+        sys.stderr.write("  (no changes needed; already in desired state)\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as e:
+        sys.stderr.write(f"ERROR: {e}\n")
+        raise SystemExit(2)
+

files/rabbit-client.sh

@@ -109,8 +109,15 @@ dispatch_task() {
   local -a extra_nbplay_opts=()
   local n=""
 
+  # NEW: allow trailing "_force" suffix (can be combined with other suffixes like _tonight / _<hours>)
+  if [[ "$base_task" =~ ^(.+)_force$ ]]; then
+    base_task="${BASH_REMATCH[1]}"
+    extra_nbplay_opts+=("-eforce_upgrade=yes")
+    log "Parsed _force suffix: base='${base_task}' (passed as -e force_upgrade=yes)"
+  fi
+
   # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4
-  if [[ "$task" =~ ^(.+)_tonight$ ]]; then
+  if [[ "$base_task" =~ ^(.+)_tonight$ ]]; then
     base_task="${BASH_REMATCH[1]}"
 
     # now, today 01:00, tomorrow 01:00 (local time)
@@ -131,20 +138,25 @@ dispatch_task() {
 
     extra_nbplay_opts+=("-erebootin=${total_h}")
     log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)"
-  elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
+  elif [[ "$base_task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then
     n="${BASH_REMATCH[1]}"
     base_task="update-rebootin"
     extra_nbplay_opts+=("-erebootin=${n}")
-  elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
+  elif [[ "$base_task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then
     n="${BASH_REMATCH[1]}"
     base_task="update-reboot"          # runs update-reboot.yml (wrapper -> update-rebootin222.yml)
     extra_nbplay_opts+=("-erebootin=${n}")
-  elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
+  elif [[ "$base_task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then
     # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin=<n>
     n="${BASH_REMATCH[1]}"
     base_task="update-indoor"
     extra_nbplay_opts+=("-erebootin=${n}")
     log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})"
+  elif [[ "$base_task" =~ ^update-indoor-bootenv_([0-9]{1,4})$ ]]; then
+    n="${BASH_REMATCH[1]}"
+    base_task="update-indoor-bootenv"
+    extra_nbplay_opts+=("-erebootin=${n}")
+    log "Parsed update-indoor-bootenv suffix: ${n}h (passed as -e rebootin=${n})"
   fi
 
   # --- Default rebootin for reboot-family when not explicitly provided ---

files/startblock.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+set -eu
+
+/bin/sh /root/stopblock.sh >/dev/null 2>&1 || true
+sleep 1
+#grep -q "block" /etc/rc.local || cat /root/rc.local-with > /etc/rc.local
+grep -qi "block" /etc/rc.local || sed -i '/^exit 0$/i\/root/startblock.sh' /etc/rc.local
+
+sleep 1
+echo 10 > /proc/sys/net/netfilter/nf_conntrack_max ; sleep 4 ; echo 7000 > /proc/sys/net/netfilter/nf_conntrack_max
+
+LAN_IF="br-lan"
+LAN_NET="192.168.2.0/24"
+EXCLUDE_IP="13.244.149.112/32"
+
+PORTAL_IP="102.38.126.180"
+PORTAL_HTTP_PORT="8082"
+PORTAL_HTTPS_PORT="8083"
+
+# -------------------------
+# NAT CAPTURE
+# -------------------------
+
+iptables -t nat -N CAPTIVE 2>/dev/null || true
+iptables -t nat -F CAPTIVE
+
+while iptables -t nat -C PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE 2>/dev/null; do
+  iptables -t nat -D PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE
+done
+iptables -t nat -I PREROUTING 1 -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE
+
+iptables -t nat -A CAPTIVE -d "$EXCLUDE_IP" -j RETURN
+
+iptables -t nat -A CAPTIVE -p udp --dport 53 -j REDIRECT --to-ports 53
+iptables -t nat -A CAPTIVE -p tcp --dport 53 -j REDIRECT --to-ports 53
+
+iptables -t nat -A CAPTIVE -p tcp --dport 80  -j DNAT --to-destination ${PORTAL_IP}:${PORTAL_HTTP_PORT}
+iptables -t nat -A CAPTIVE -p tcp --dport 443 -j DNAT --to-destination ${PORTAL_IP}:${PORTAL_HTTPS_PORT}
+
+# -------------------------
+# FILTER ENFORCEMENT (BLOCK EVERYTHING ELSE)
+# -------------------------
+
+iptables -N CAPTIVE_BLOCK 2>/dev/null || true
+iptables -F CAPTIVE_BLOCK
+
+# Allow established traffic
+iptables -A CAPTIVE_BLOCK -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+
+# Allow DNS to router itself (client -> router)
+iptables -A CAPTIVE_BLOCK -p udp -d 192.168.2.1 --dport 53 -j ACCEPT
+iptables -A CAPTIVE_BLOCK -p tcp -d 192.168.2.1 --dport 53 -j ACCEPT
+
+# Allow access to captive portal server (after DNAT)
+iptables -A CAPTIVE_BLOCK -p tcp -d "$PORTAL_IP" --dport "$PORTAL_HTTP_PORT" -j ACCEPT
+iptables -A CAPTIVE_BLOCK -p tcp -d "$PORTAL_IP" --dport "$PORTAL_HTTPS_PORT" -j ACCEPT
+
+# Allow bypass destination fully (optional but usually desired)
+iptables -A CAPTIVE_BLOCK -d "$EXCLUDE_IP" -j ACCEPT
+
+# Block everything else from LAN clients
+iptables -A CAPTIVE_BLOCK -j REJECT --reject-with icmp-admin-prohibited
+
+# Hook CAPTIVE_BLOCK into FORWARD (ensure single jump)
+while iptables -C FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK 2>/dev/null; do
+  iptables -D FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK
+done
+iptables -I FORWARD 1 -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK
+
+echo "OK: filter rules applied"
+

files/stopblock.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+set -eu
+#cat /root/rc.local-with | grep -v block > /etc/rc.local
+
+grep -qi "block" /etc/rc.local && sed -i '\|^/root/startblock\.sh$|d' /etc/rc.local
+
+LAN_IF="br-lan"
+LAN_NET="192.168.2.0/24"
+
+# -------------------------
+# FILTER: remove CAPTIVE_BLOCK hook + chain
+# -------------------------
+
+# Remove any FORWARD jumps that match our hook exactly
+while iptables -C FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK 2>/dev/null; do
+  iptables -D FORWARD -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE_BLOCK
+done
+
+# Flush/delete CAPTIVE_BLOCK chain if present
+iptables -F CAPTIVE_BLOCK 2>/dev/null || true
+iptables -X CAPTIVE_BLOCK 2>/dev/null || true
+
+# -------------------------
+# NAT: remove CAPTIVE hook + chain
+# -------------------------
+
+# Remove any PREROUTING jumps that match our hook exactly
+while iptables -t nat -C PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE 2>/dev/null; do
+  iptables -t nat -D PREROUTING -i "$LAN_IF" -s "$LAN_NET" -j CAPTIVE
+done
+
+# Flush/delete CAPTIVE chain if present
+iptables -t nat -F CAPTIVE 2>/dev/null || true
+iptables -t nat -X CAPTIVE 2>/dev/null || true
+
+echo "OK: filter rules removed"