1246

files/ansible-playbooks/afterupgrade_check.yml-from_chat

@@ -1,355 +0,0 @@
----
-- name: After-upgrade verification (banner check + reporting)
-  hosts: all
-  gather_facts: no
-
-  # RabbitMQ + defaults (match the big script)
-  vars:
-    rmq_host: "10.210.12.2"
-    rmq_port: 15672
-    rmq_user: "admin"
-    rmq_pass: "change_me"
-    rmq_vhost: "app"
-    rmq_exchange: "controls"
-    control_queue: "queue_controls"
-
-    # Probing/SSH defaults
-    tcp_port: 22
-    nc_timeout: 5
-    ssh_user: "{{ ansible_user | default('root') }}"
-    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
-    ssh_timeout: 10
-
-    # Do NOT self-reference max_attempts. We’ll normalize below.
-    max_attempts_default: 3
-
-  tasks:
-    # ---- Normalize metadata safely (no self-referential defaults) ----
-    - name: Normalize metadata (no clever transforms)
-      ansible.builtin.set_fact:
-        attempt: "{{ (attempt | default(1)) | int }}"
-        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
-        correlation_id: "{{ correlation_id | default('') }}"
-        original_emitted_at: "{{ original_emitted_at | default('') }}"
-        target_version: "{{ target_version | default('') }}"
-        # preserve the original string verbatim for all subsequent retries
-        target_version_full: "{{ target_version | default('') }}"
-
-    - name: Show received metadata
-      ansible.builtin.debug:
-        msg:
-          - "attempt={{ attempt }}"
-          - "max_attempts={{ effective_max_attempts }}"
-          - "correlation_id={{ correlation_id }}"
-          - "original_emitted_at={{ original_emitted_at }}"
-          - "target_version(full)={{ target_version_full }}"
-
-    # ---- Fast TCP reachability probe (controller-side) ----
-    - name: Check if TCP/{{ tcp_port }} is reachable with nc
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
-      register: nc_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Build failure journal (no TCP connectivity) + mark retry
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-          _needs_retry: true
-
-    - name: Publish failure journal (no TCP connectivity)
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_tcp_fail
-      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
-
-    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
-    - name: Stop host after TCP failure (we’ll schedule or close out below)
-      when: nc_probe.rc != 0
-      ansible.builtin.meta: noop
-
-    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
-    - name: Probe banner via SSH from controller (classic extraction)
-      when: nc_probe.rc == 0
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        USER="{{ ssh_user }}"
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        sshpass -p '{{ ssh_pass }}' \
-        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
-          "${USER}@${HOST}" \
-          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
-      register: banner_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Show the current version (banner line)
-      when: nc_probe.rc == 0 and banner_probe.rc == 0
-      ansible.builtin.debug:
-        msg: "{{ banner_probe.stdout | trim }}"
-
-    - name: Evaluate version match (full-string contains check)
-      when: nc_probe.rc == 0 and banner_probe.rc == 0
-      ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
-
-    # ---- Journaling paths ----
-    # Success: banner matches expected full target_version
-    - name: Build success journal payload
-      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_success_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
-
-    - name: Publish success journal to control queue
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_success_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_success
-      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
-
-    # NEW: send a control tag to clean up device state on success
-    - name: Build cleanup control payload (update_cleanup_success)
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        control_cleanup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "update_cleanup_success"
-
-    - name: Publish cleanup control message to control queue
-      when: control_cleanup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ control_cleanup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_cleanup_success
-      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
-
-    # Mismatch: reachable & banner read, but not equal to target_version
-    - name: Build mismatch journal payload
-      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_mismatch_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
-
-    - name: Publish mismatch journal to control queue
-      when: journal_mismatch_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_mismatch_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_mismatch
-      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
-
-    # SSH error path: TCP OK, but SSH failed
-    - name: Build failure journal payload (ssh error) + mark retry
-      when: nc_probe.rc == 0 and banner_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            {{ (banner_probe.stderr | default('') | trim) }}
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-          _needs_retry: true
-
-    - name: Publish failure journal (ssh error) to control queue
-      when: nc_probe.rc == 0 and banner_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_fail
-      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
-
-    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
-    - name: Compute next-attempt delay (ms) according to policy
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.set_fact:
-        next_attempt: "{{ attempt | int + 1 }}"
-        next_delay_sec: >-
-          {% if attempt | int == 1 %}
-          300
-          {% elif attempt | int == 2 %}
-          600
-          {% else %}
-          0
-          {% endif %}
-        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
-
-    # If we've reached the cap, send a final “gave up” journal and stop.
-    - name: Build final gave-up journal (max attempts reached)
-      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_gaveup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-
-    - name: Publish final gave-up journal
-      when: journal_gaveup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_gaveup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_gaveup
-      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
-
-    - name: Stop host after final gave-up
-      when: journal_gaveup_payload is defined
-      ansible.builtin.meta: end_host
-
-    # Otherwise schedule the next attempt (only if we still have budget)
-    - name: Build delayed after-upgrade payload for next attempt
-      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        delayed_payload:
-          task_name: "afterupgrade_check"
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          attempt: "{{ next_attempt | int }}"
-          max_attempts: "{{ effective_max_attempts | int }}"
-          correlation_id: "{{ correlation_id }}"
-          original_emitted_at: "{{ original_emitted_at }}"
-          target_version: "{{ target_version_full }}"
-          current_delay_sec: "{{ next_delay_sec | int }}"
-          schema_version: 1
-
-    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
-      when: delayed_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          expiration: "{{ (next_delay_ms | int) | string }}"
-          correlation_id: "{{ correlation_id }}"
-          routing_key: "deviceconfig"
-          payload: "{{ delayed_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_next
-      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
-
-    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_check.yml-from_container

@@ -1,432 +0,0 @@
----
-- name: After-upgrade verification (banner check + reporting)
-  hosts: all
-  gather_facts: no
-
-  # RabbitMQ + defaults (match the big script)
-  vars:
-    rmq_host: "10.210.12.2"
-    rmq_port: 15672
-    rmq_user: "admin"
-    rmq_pass: "change_me"
-    rmq_vhost: "app"
-    rmq_exchange: "controls"
-    control_queue: "queue_controls"
-
-    # Probing/SSH defaults
-    tcp_port: 22
-    nc_timeout: 5
-    ssh_user: "{{ ansible_user | default('root') }}"
-    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
-    ssh_timeout: 10
-
-    # Do NOT self-reference max_attempts. We’ll normalize below.
-    max_attempts_default: 3
-
-  tasks:
-    # ---- Normalize metadata safely (no self-referential defaults) ----
-    - name: Normalize metadata (no clever transforms)
-      ansible.builtin.set_fact:
-        attempt: "{{ (attempt | default(1)) | int }}"
-        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
-        correlation_id: "{{ correlation_id | default('') }}"
-        original_emitted_at: "{{ original_emitted_at | default('') }}"
-        target_version: "{{ target_version | default('') }}"
-        # preserve the original string verbatim for all subsequent retries
-        target_version_full: "{{ target_version | default('') }}"
-
-    - name: Show received metadata
-      ansible.builtin.debug:
-        msg:
-          - "attempt={{ attempt }}"
-          - "max_attempts={{ effective_max_attempts }}"
-          - "correlation_id={{ correlation_id }}"
-          - "original_emitted_at={{ original_emitted_at }}"
-          - "target_version(full)={{ target_version_full }}"
-
-    # ---- Fast TCP reachability probe (controller-side) ----
-    - name: Check if TCP/{{ tcp_port }} is reachable with nc
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
-      register: nc_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Build failure journal (no TCP connectivity) + mark retry
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
-            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-        _needs_retry: true
-
-    - name: Publish failure journal (no TCP connectivity)
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_tcp_fail
-      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
-
-    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
-    - name: Stop host after TCP failure (we’ll schedule or close out below)
-      when: nc_probe.rc != 0
-      ansible.builtin.meta: noop
-
-    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
-    - name: Probe banner via SSH from controller (classic extraction)
-      when: nc_probe.rc == 0
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        USER="{{ ssh_user }}"
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        sshpass -p '{{ ssh_pass }}' \
-        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
-          "${USER}@${HOST}" \
-          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
-      register: banner_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Show the current version (banner line)
-      when: nc_probe.rc == 0 and banner_probe.rc == 0
-      ansible.builtin.debug:
-        msg: "{{ banner_probe.stdout | trim }}"
-
-    - name: Evaluate version match (full-string contains check)
-      when: nc_probe.rc == 0 and banner_probe.rc == 0
-      ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
-
-    # ---- Journaling paths ----
-
-    # Success: banner matches expected full target_version
-    - name: Build success journal payload
-      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_success_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}'
-            Target='{{ target_version_full }}'
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
-
-    - name: Publish success journal to control queue
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_success_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_success
-      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
-
-    # NEW: send a control tag to clean up device state on success
-    - name: Build cleanup control payload (update_cleanup_success)
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        control_cleanup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "update_cleanup_success"
-
-    - name: Publish cleanup control message to control queue
-      when: control_cleanup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ control_cleanup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_cleanup_success
-      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
-
-    # --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag ---
-
-    - name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801")
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        indoor_fwver_norm: >-
-          {{
-            ( (dev2_fwver.stdout | default('') | trim)
-              | regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1')
-              | regex_replace('\\s+', ' ')
-              | trim
-            )
-          }}
-
-    - name: Publish NetBox custom field indoor_fwver to control queue
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers: { content-type: "application/json" }
-        body_format: json
-        body:
-          properties: { content_type: "application/json" }
-          routing_key: "{{ control_queue }}"
-          payload: "{{ {
-            'inscope_device': (ansible_hostname | default(inventory_hostname)),
-            'task_name': 'custom_field_set',
-            'task_add1': 'indoor_fwver',
-            'task_result': (indoor_fwver_norm | default(''))
-          } | to_json }}"
-          payload_encoding: "string"
-      register: rmq_cf_indoor_fwver
-      changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool)
-
-    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        tag_remove_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "tag_remove"
-          task_result: "indoor-restart-scheduled"
-
-    - name: Publish indoor-restart-scheduled tag removal
-      when: tag_remove_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ tag_remove_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_tag_remove_sched
-      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
-
-
-    # Mismatch: reachable & banner read, but not equal to target_version
-    - name: Build mismatch journal payload
-      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_mismatch_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
-
-    - name: Publish mismatch journal to control queue
-      when: journal_mismatch_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_mismatch_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_mismatch
-      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
-
-    # SSH error path: TCP OK, but SSH failed
-    - name: Build failure journal payload (ssh error) + mark retry
-      when: nc_probe.rc == 0 and banner_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            {{ (banner_probe.stderr | default('') | trim) }}
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-        _needs_retry: true
-
-    - name: Publish failure journal (ssh error) to control queue
-      when: nc_probe.rc == 0 and banner_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_fail
-      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
-
-    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
-
-    - name: Compute next-attempt delay (ms) according to policy
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.set_fact:
-        next_attempt: "{{ attempt | int + 1 }}"
-        next_delay_sec: >-
-          {% if attempt | int == 1 %}
-          300
-          {% elif attempt | int == 2 %}
-          600
-          {% else %}
-          0
-          {% endif %}
-        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
-
-    # If we've reached the cap, send a final “gave up” journal and stop.
-    - name: Build final gave-up journal (max attempts reached)
-      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_gaveup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-
-    - name: Publish final gave-up journal
-      when: journal_gaveup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_gaveup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_gaveup
-      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
-
-    - name: Stop host after final gave-up
-      when: journal_gaveup_payload is defined
-      ansible.builtin.meta: end_host
-
-    # Otherwise schedule the next attempt (only if we still have budget)
-    - name: Build delayed after-upgrade payload for next attempt
-      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        delayed_payload:
-          task_name: "afterupgrade_check"
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          attempt: "{{ next_attempt | int }}"
-          max_attempts: "{{ effective_max_attempts | int }}"
-          correlation_id: "{{ correlation_id }}"
-          original_emitted_at: "{{ original_emitted_at }}"
-          target_version: "{{ target_version_full }}"
-          current_delay_sec: "{{ next_delay_sec | int }}"
-          schema_version: 1
-
-    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
-      when: delayed_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-            expiration: "{{ (next_delay_ms | int) | string }}"
-            correlation_id: "{{ correlation_id }}"
-          routing_key: "deviceconfig"
-          payload: "{{ delayed_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_next
-      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
-
-    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.meta: end_host
-

files/ansible-playbooks/afterupgrade_check.yml-screewed

@@ -1,359 +0,0 @@
----
-- name: After-upgrade verification (banner check + reporting)
-  hosts: all
-  gather_facts: no
-
-  # RabbitMQ + defaults (match the big script)
-  vars:
-    rmq_host: "10.210.12.2"
-    rmq_port: 15672
-    rmq_user: "admin"
-    rmq_pass: "change_me"
-    rmq_vhost: "app"
-    rmq_exchange: "controls"
-    control_queue: "queue_controls"
-
-    # Probing/SSH defaults
-    tcp_port: 22
-    nc_timeout: 5
-    ssh_user: "{{ ansible_user | default('root') }}"
-    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
-    ssh_timeout: 10
-
-    # Do NOT self-reference max_attempts. We’ll normalize below.
-    max_attempts_default: 3
-
-  tasks:
-    # ---- Normalize metadata safely (no self-referential defaults) ----
-    - name: Normalize metadata (no clever transforms)
-      ansible.builtin.set_fact:
-        attempt: "{{ (attempt | default(1)) | int }}"
-        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
-        correlation_id: "{{ correlation_id | default('') }}"
-        original_emitted_at: "{{ original_emitted_at | default('') }}"
-        target_version: "{{ target_version | default('') }}"
-        # preserve the original string verbatim for all subsequent retries
-        target_version_full: "{{ target_version | default('') }}"
-
-    - name: Show received metadata
-      ansible.builtin.debug:
-        msg:
-          - "attempt={{ attempt }}"
-          - "max_attempts={{ effective_max_attempts }}"
-          - "correlation_id={{ correlation_id }}"
-          - "original_emitted_at={{ original_emitted_at }}"
-          - "target_version(full)={{ target_version_full }}"
-
-    # ---- Fast TCP reachability probe (controller-side) ----
-    - name: Check if TCP/{{ tcp_port }} is reachable with nc
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
-      register: nc_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Build failure journal (no TCP connectivity) + mark retry
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
-            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-        _needs_retry: true
-
-    - name: Publish failure journal (no TCP connectivity)
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_tcp_fail
-      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
-
-    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
-    - name: Stop host after TCP failure (we’ll schedule or close out below)
-      when: nc_probe.rc != 0
-      ansible.builtin.meta: noop
-
-    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
-    - name: Probe banner via SSH from controller (classic extraction)
-      when: nc_probe.rc == 0
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        USER="{{ ssh_user }}"
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        sshpass -p '{{ ssh_pass }}' \
-        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
-          "${USER}@${HOST}" \
-          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
-      register: banner_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Show the current version (banner line)
-      when: nc_probe.rc == 0 and banner_probe.rc == 0
-      ansible.builtin.debug:
-        msg: "{{ banner_probe.stdout | trim }}"
-
-    - name: Evaluate version match (full-string contains check)
-      when: nc_probe.rc == 0 and banner_probe.rc == 0
-      ansible.builtin.set_fact:
-        version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
-
-    # ---- Journaling paths ----
-
-    # Success: banner matches expected full target_version
-    - name: Build success journal payload
-      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_success_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Banner='{{ (banner_probe.stdout | default('') | trim) }}'
-            Target='{{ target_version_full }}'
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
-
-    - name: Publish success journal to control queue
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_success_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_success
-      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
-
-    # NEW: send a control tag to clean up device state on success
-    - name: Build cleanup control payload (update_cleanup_success)
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        control_cleanup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "update_cleanup_success"
-
-    - name: Publish cleanup control message to control queue
-      when: control_cleanup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ control_cleanup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_cleanup_success
-      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
-
-    # Mismatch: reachable & banner read, but not equal to target_version
-    - name: Build mismatch journal payload
-      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_mismatch_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
-
-    - name: Publish mismatch journal to control queue
-      when: journal_mismatch_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_mismatch_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_mismatch
-      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
-
-    # SSH error path: TCP OK, but SSH failed
-    - name: Build failure journal payload (ssh error) + mark retry
-      when: nc_probe.rc == 0 and banner_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            {{ (banner_probe.stderr | default('') | trim) }}
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-        _needs_retry: true
-
-    - name: Publish failure journal (ssh error) to control queue
-      when: nc_probe.rc == 0 and banner_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_fail
-      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
-
-    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
-
-    - name: Compute next-attempt delay (ms) according to policy
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.set_fact:
-        next_attempt: "{{ attempt | int + 1 }}"
-        next_delay_sec: >-
-          {% if attempt | int == 1 %}
-          300
-          {% elif attempt | int == 2 %}
-          600
-          {% else %}
-          0
-          {% endif %}
-        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
-
-    # If we've reached the cap, send a final “gave up” journal and stop.
-    - name: Build final gave-up journal (max attempts reached)
-      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_gaveup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-
-    - name: Publish final gave-up journal
-      when: journal_gaveup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_gaveup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_gaveup
-      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
-
-    - name: Stop host after final gave-up
-      when: journal_gaveup_payload is defined
-      ansible.builtin.meta: end_host
-
-    # Otherwise schedule the next attempt (only if we still have budget)
-    - name: Build delayed after-upgrade payload for next attempt
-      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        delayed_payload:
-          task_name: "afterupgrade_check"
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          attempt: "{{ next_attempt | int }}"
-          max_attempts: "{{ effective_max_attempts | int }}"
-          correlation_id: "{{ correlation_id }}"
-          original_emitted_at: "{{ original_emitted_at }}"
-          target_version: "{{ target_version_full }}"
-          current_delay_sec: "{{ next_delay_sec | int }}"
-          schema_version: 1
-
-    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
-      when: delayed_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-            expiration: "{{ (next_delay_ms | int) | string }}"
-            correlation_id: "{{ correlation_id }}"
-          routing_key: "deviceconfig"
-          payload: "{{ delayed_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_next
-      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
-
-    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.meta: end_host

files/ansible-playbooks/afterupgrade_indoor_check.yml

@@ -1,5 +1,5 @@
 ---
-- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
+- name: After upgrade verification for indoor DEV2 via DEV1 tunnel
   hosts: all
   gather_facts: no
 
@@ -30,20 +30,22 @@
     rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
 
+    debug_aic: false
+
   tasks:
-    # ---- Normalize metadata (from delayed message) ----
-    - name: Normalize after-upgrade metadata
+    # ---- Normalize metadata from delayed message ----
+    - name: Normalize after upgrade metadata into facts
       delegate_to: localhost
       ansible.builtin.set_fact:
         attempt: "{{ (attempt | default(1)) | int }}"
         effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
         correlation_id: "{{ correlation_id | default('') }}"
         original_emitted_at: "{{ original_emitted_at | default('') }}"
+        # keep whatever scheduler sent in core and full; prefer full when available
         target_version: "{{ target_version | default('') }}"
-        target_version_full: "{{ target_version | default('') }}"
+        target_version_full: "{{ target_version_full | default(target_version | default('')) }}"
 
-    # (NEW) Show what we received from the scheduler (for easy troubleshooting)
-    - name: Debug received scheduler metadata
+    - name: Debug show received scheduler metadata
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
@@ -51,10 +53,23 @@
           - "max_attempts={{ effective_max_attempts }}"
           - "correlation_id={{ correlation_id }}"
           - "original_emitted_at={{ original_emitted_at }}"
-          - "target_version(full)={{ target_version_full }}"
+          - "target_version_full={{ target_version_full }}"
+          - "target_version={{ target_version | default('') }}"
 
-    # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
-    - name: Check if TCP/22 on DEV1 is reachable
+
+    # Visibility of what scheduler sent
+    - name: Debug show received scheduler metadata
+      delegate_to: localhost
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt }}"
+          - "max_attempts={{ effective_max_attempts }}"
+          - "correlation_id={{ correlation_id }}"
+          - "original_emitted_at={{ original_emitted_at }}"
+          - "target_version_full={{ target_version_full }}"
+
+    # ---- Controller side TCP probe to DEV1 ----
+    - name: Probe DEV1 TCP 22 from controller
       delegate_to: localhost
       ansible.builtin.shell: |
         nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
@@ -62,8 +77,8 @@
       changed_when: false
       ignore_errors: true
 
-    # ---- If TCP down: journal + schedule next try or give up ----
-    - name: Build failure journal (TCP unreachable) + mark retry
+    # ---- If TCP down publish journal and schedule next try or give up ----
+    - name: Build journal payload for DEV1 TCP unreachable and mark retry
       when: nc_probe.rc != 0
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -71,12 +86,12 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "journal_add"
           task_result: >-
-            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }})
             TCP 22 unreachable. Correlation={{ correlation_id }}
             Original={{ original_emitted_at }} Target='{{ target_version_full }}'
         _needs_retry: true
 
-    - name: Publish failure journal (TCP unreachable)
+    - name: Publish journal for DEV1 TCP unreachable
       when: nc_probe.rc != 0
       delegate_to: localhost
       ansible.builtin.uri:
@@ -96,7 +111,7 @@
       register: rmq_j_tcp_fail
       changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
 
-    - name: Compute next-attempt delay (10 minutes) and counters
+    - name: Compute next attempt delay ten minutes and counters
       when: (_needs_retry | default(false)) | bool
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -104,7 +119,7 @@
         next_delay_sec: 600
         next_delay_ms: 600000
 
-    - name: Build final gave-up journal (max attempts reached)
+    - name: Build final gave up journal when max attempts reached
       when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -112,11 +127,11 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "journal_add"
           task_result: >-
-            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }})
             Exhausted attempts. Last error path=TCP.
             Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
 
-    - name: Publish final gave-up journal
+    - name: Publish final gave up journal after TCP failures
       when: journal_gaveup_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -136,11 +151,11 @@
       register: rmq_j_gaveup
       changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
 
-    - name: Stop host after final gave-up
+    - name: Stop host after final gave up due to TCP failure
       when: journal_gaveup_payload is defined
       ansible.builtin.meta: end_host
 
-    - name: Build delayed after-upgrade payload for next attempt (10m)
+    - name: Build delayed payload for next attempt after upgrade ten minutes
       when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -151,11 +166,12 @@
           max_attempts: "{{ effective_max_attempts | int }}"
           correlation_id: "{{ correlation_id }}"
           original_emitted_at: "{{ original_emitted_at }}"
-          target_version: "{{ target_version_full }}"
+          target_version_full: "{{ target_version_full }}"     
+          target_version: "{{ target_version | default(target_version_full) }}"  
           current_delay_sec: "{{ next_delay_sec | int }}"
           schema_version: 1
 
-    - name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
+    - name: Publish delayed next attempt to holding exchange with TTL to deviceconfig
       when: delayed_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -183,8 +199,8 @@
       ansible.builtin.meta: end_host
 
 
-    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
-    - name: Read DEV1 hostname
+    # ---------------- Step 1 Read DEV1 hostname and sanity ----------------
+    - name: Read DEV1 hostname from kernel procfs
       when: nc_probe.rc == 0
       ansible.builtin.raw: >
         {{ pathprefix }}
@@ -192,13 +208,13 @@
       register: dev1_host_read
       changed_when: false
 
-    - name: Stop if DEV1 hostname mismatch
+    - name: Stop if DEV1 hostname does not match inventory hostname
       ansible.builtin.meta: end_host
       when: (dev1_host_read.stdout | trim | length > 0) and
             ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
 
-    # ---------------- Step 2: Setup temporary IP for reachability ----------------
-    - name: Add temporary IP on DEV1 (ignore if exists)
+    # ---------------- Step 2 Setup temporary IP for reachability ----------------
+    - name: Add temporary IP on DEV1 ignore if address already exists
       ansible.builtin.raw: >
         {{ pathprefix }}
         ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
@@ -209,8 +225,8 @@
         ('File exists' not in (add_ip.stdout | default(''))) and
         ('File exists' not in (add_ip.stderr | default('')))
 
-    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
-    - name: Pick random free local port
+    # ---------------- Step 3 Start SSH tunnel via DEV1 ----------------
+    - name: Pick random free local port for SSH tunnel
       delegate_to: localhost
       ansible.builtin.shell: |
         set -e
@@ -222,22 +238,22 @@
       register: pick_port
       changed_when: false
 
-    - name: Stop if no free local port
+    - name: Stop if no free local port was found
       ansible.builtin.meta: end_host
       when: (pick_port.stdout | trim | length) == 0
 
-    - name: Record chosen local port and control dir
+    - name: Record chosen local port and create control directory
       delegate_to: localhost
       ansible.builtin.set_fact:
         _local_port: "{{ pick_port.stdout | trim }}"
         _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
 
-    - name: Build tunnel control socket path
+    - name: Build SSH control socket path for tunnel
       delegate_to: localhost
       ansible.builtin.set_fact:
         _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
 
-    - name: Start SSH ControlMaster tunnel via DEV1
+    - name: Start SSH ControlMaster tunnel via DEV1 to DEV2
       delegate_to: localhost
       ansible.builtin.shell: |
         set -e
@@ -251,14 +267,14 @@
       register: start_tunnel
       changed_when: true
 
-    # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
-    - name: Small delay for tunnel to settle
+    # --- settle and check ControlMaster and TCP probe ---
+    - name: Pause briefly to let SSH tunnel settle
       delegate_to: localhost
       ansible.builtin.wait_for:
         timeout: 1
       changed_when: false
 
-    - name: Verify tunnel master running (ssh -O check)
+    - name: Verify tunnel ControlMaster is running with ssh O check
       delegate_to: localhost
       ansible.builtin.shell: |
         set -e
@@ -267,14 +283,14 @@
       register: tun_check
       changed_when: false
 
-    - name: Debug tunnel check
+    - name: Debug show tunnel master check result
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
           - "tunnel_check.rc={{ tun_check.rc }}"
           - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
 
-    - name: Sanity confirm tunnel TCP reachability to DEV2
+    - name: Probe TCP reachability through tunnel to DEV2
       delegate_to: localhost
       ansible.builtin.shell: |
         set -e
@@ -283,7 +299,7 @@
       changed_when: false
       ignore_errors: true
 
-    - name: Debug reachability result
+    - name: Debug show reachability probe result through tunnel
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
@@ -291,12 +307,12 @@
           - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
           - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
 
-    - name: Stop if tunnel TCP check failed
+    - name: Stop if tunnel reachability check failed
       ansible.builtin.meta: end_host
       when: nc_probe.rc != 0
 
-    # ---------------- Step 4: Determine working password for DEV2 ----------------
-    - name: Try both passfiles for DEV2
+    # ---------------- Step 4 Determine working password for DEV2 ----------------
+    - name: Try provided passfiles to authenticate to DEV2
       delegate_to: localhost
       ansible.builtin.shell: |
         for f in {{ dev2_passfiles | join(' ') }}; do
@@ -308,23 +324,23 @@
       register: dev2_passfile_try
       changed_when: false
 
-    - name: Save selected DEV2 passfile
+    - name: Save selected DEV2 passfile for subsequent commands
       delegate_to: localhost
       ansible.builtin.set_fact:
         dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
       changed_when: false
 
-    - name: Debug selected DEV2 passfile
+    - name: Debug show which DEV2 passfile was selected
       delegate_to: localhost
       ansible.builtin.debug:
         msg: "dev2_passfile_used={{ dev2_passfile_used }}"
 
-    - name: Stop if no valid passfile found
+    - name: Stop if no valid DEV2 passfile was found
       ansible.builtin.meta: end_host
       when: dev2_passfile_used == "NONE"
 
-    # ---------------- Step 5: Read firmware version on DEV2 ----------------
-    - name: Read firmware version from DEV2
+    # ---------------- Step 5 Read firmware version on DEV2 ----------------
+    - name: Read firmware version or banner from DEV2
       delegate_to: localhost
       ansible.builtin.shell: |
         sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
@@ -334,58 +350,114 @@
       changed_when: false
       ignore_errors: true
 
-    - name: Show firmware version readout
+    - name: Debug | banner normalization inputs
+      delegate_to: localhost
+      when: (debug_aic | default(false)) | bool
+      ansible.builtin.debug:
+        msg:
+          - "banner_raw={{ (dev2_fwver.stdout | default('') | trim) }}"
+          - "normalize rule: '2.2.1 rev 6801' -> '2.2.1-r6801'"
+
+    - name: Debug show raw firmware readout from DEV2
       delegate_to: localhost
       ansible.builtin.debug:
-        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
+        msg: "Firmware version on DEV2 {{ (dev2_fwver.stdout | default('')) | trim }}"
 
-    # ---------------- Retry metadata + success evaluation (read OK) ----------------
-    - name: Normalize retry metadata for indoor checker
+    # ---------------- Retry metadata and success evaluation ----------------
+    - name: Normalize retry control facts for indoor checker
       ansible.builtin.set_fact:
         attempt: "{{ (attempt | default(1)) | int }}"
         effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
         correlation_id: "{{ correlation_id | default('') }}"
         original_emitted_at: "{{ original_emitted_at | default('') }}"
 
-    - name: Check if firmware read succeeded (read_ok)
+    - name: Compute read_ok flag based on firmware readout
       delegate_to: localhost
       ansible.builtin.set_fact:
         read_ok: "{{ (dev2_fwver.rc | default(1) == 0)
                     and ((dev2_fwver.stdout | default('') | trim) | length > 0)
-                    and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
+                    and (not (((dev2_fwver.stdout | default('unknown')) | lower) is search('unknown'))) }}"
 
-    # ===================== (NEW) Unconditional normalization + comparison =====================
-    # Compute expected_norm from target_version_full. If the full filename-like string is sent,
-    # we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original.
+    # ===================== Normalization and comparison =====================
+    # Compute expected_norm from target_version_full. If a whole filename is sent,
+    # extract X.Y.Z-rNNNN otherwise keep the original trimmed string.
 
-    - name: Normalize expected target string (step 1: compute components)
+    - name: Normalize expected target step one compute base string
       delegate_to: localhost
       ansible.builtin.set_fact:
         expected_norm_step1: "{{ (target_version_full | default('') | trim) }}"
-        expected_norm_core: >-
-          {{
-            (target_version_full | default('') |
-              regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1'))
-            | default('', true)
-          }}
 
-    - name: Normalize expected target string (step 2: choose core if present)
+    - name: Debug | expected target base and regex extraction inputs
+      delegate_to: localhost
+      when: debug_aic | bool
+      ansible.builtin.debug:
+        msg:
+          - "target_version(raw)={{ target_version | default('') }}"
+          - "target_version_full(raw)={{ target_version_full | default('') }}"
+          - "regex='[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+'"
+
+
+    - name: Extract version core X dot Y dot Z dash rNNNN from target if present
       delegate_to: localhost
       ansible.builtin.set_fact:
-        expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+        expected_norm_core_list: "{{ (target_version_full | default('') | regex_findall('[0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+')) | default([]) }}"
 
-    # Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801"
-    - name: Normalize banner/firmux string from DEV2
+    - name: Choose first extracted core if available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm_core: "{{ (expected_norm_core_list | default([]) | length > 0) | ternary((expected_norm_core_list | first), '') }}"
+
+    - name: Normalize expected target step two prefer extracted core when available
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        expected_norm: "{{ (expected_norm_core | default('') | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}"
+
+    - name: Debug | final expected normalized version (what we will compare against)
+      delegate_to: localhost
+      when: debug_aic | bool
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm={{ expected_norm | default('') }}"
+
+
+    # 1 capture raw banner line from DEV2
+    - name: Capture firmware banner line from DEV2
       delegate_to: localhost
       ansible.builtin.set_fact:
         banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}"
+
+    # 2 normalize rev suffix to dash rNNNN
+    - name: Normalize banner line to X dot Y dot Z dash rNNNN suffix
+      delegate_to: localhost
+      ansible.builtin.set_fact:
         banner_norm: >-
           {{
-            (banner_raw | lower is search('-r[0-9]+$'))
-            | ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')))
+            ((banner_raw | lower) is search('-r[0-9]+$'))
+            | ternary(
+                banner_raw,
+                (banner_raw | regex_replace('\s*[Rr][Ee][Vv]\.?\s*([0-9]+)\s*$', '-r\1'))
+              )
           }}
 
-    - name: Evaluate version match (normalized equality or contains)
+    - name: Debug | banner normalized result
+      delegate_to: localhost
+      when: debug_aic | bool
+      ansible.builtin.debug:
+        msg:
+          - "banner_norm={{ banner_norm | default('') }}"
+
+    - name: Debug | version compare inputs and strategy
+      delegate_to: localhost
+      when: debug_aic | bool
+      ansible.builtin.debug:
+        msg:
+          - "expected_norm={{ expected_norm | default('') }}"
+          - "banner_norm={{ banner_norm | default('') }}"
+          - "eq={{ (expected_norm | default('')) == (banner_norm | default('')) }}"
+          - "substr={{ (banner_norm | default('')) is search((expected_norm | default('')), ignorecase=False) if (expected_norm | default('') | length) > 0 else 'N/A' }}"
+
+
+    - name: Evaluate version match using normalized equality or substring
       delegate_to: localhost
       ansible.builtin.set_fact:
         version_match: >-
@@ -393,22 +465,30 @@
             (expected_norm | length > 0)
             and (
               (banner_norm == expected_norm)
-              or (banner_norm is search(expected_norm))
-              or (expected_norm is search(banner_norm))
+              or ((banner_norm | lower) is search((expected_norm | lower)))
+              or ((expected_norm | lower) is search((banner_norm | lower)))
             )
           }}
 
-    - name: Debug compare snapshot (expected vs actual normalized)
+    - name: Debug | version match outcome
+      delegate_to: localhost
+      when: debug_aic | bool
+      ansible.builtin.debug:
+        msg:
+          - "version_match={{ version_match | default(false) }}"
+
+
+    - name: Debug snapshot of expected and actual normalized versions
       delegate_to: localhost
       ansible.builtin.debug:
         msg:
-          - "expected_norm='{{ expected_norm }}'"
-          - "banner_norm='{{ banner_norm }}'"
+          - "expected_norm={{ expected_norm }}"
+          - "banner_norm={{ banner_norm }}"
           - "version_match={{ version_match | default(false) }}"
 
-    # ===================== Journaling/Tagging paths =====================
-    # SUCCESS: read_ok AND version_match
-    - name: Build success journal payload
+    # ===================== Journaling and tagging paths =====================
+    # SUCCESS path
+    - name: Build journal payload for successful after upgrade check
       when: (read_ok | bool) and (version_match | bool)
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -416,7 +496,7 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "journal_add"
           task_result: >-
-            afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }})
             Banner='{{ banner_raw }}' Target='{{ expected_norm }}'
             Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
 
@@ -442,7 +522,7 @@
       register: rmq_pub_success
       changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
 
-    - name: Build payload to add indoor-update-success tag (DEV1)
+    - name: Build payload to add tag indoor update success on DEV1
       when: journal_success_payload is defined
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -451,7 +531,7 @@
           task_name: "tag_add"
           task_result: "indoor-update-success"
 
-    - name: Publish indoor-update-success tag
+    - name: Publish tag indoor update success
       when: tag_add_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -473,7 +553,7 @@
       register: rmq_tag_add_success
       changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
 
-    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
+    - name: Build payload to remove tag indoor restart scheduled on DEV1
       when: journal_success_payload is defined
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -482,7 +562,7 @@
           task_name: "tag_remove"
           task_result: "indoor-restart-scheduled"
 
-    - name: Publish indoor-restart-scheduled tag removal
+    - name: Publish tag removal indoor restart scheduled
       when: tag_remove_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -505,13 +585,13 @@
       changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
 
     # --- Normalize firmware string and set custom field on success ---
-    - name: Capture raw firmware banner from DEV2 (for normalization)
+    - name: Capture raw firmware banner for normalization on success path
       when: journal_success_payload is defined
       delegate_to: localhost
       ansible.builtin.set_fact:
         fw_banner_raw: "{{ banner_raw }}"
 
-    - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
+    - name: Normalize firmware string for indoor_fwver to X dot Y dot Z dash rNNNN
       when:
         - journal_success_payload is defined
         - (fw_banner_raw | default('') | length) > 0
@@ -519,18 +599,20 @@
       ansible.builtin.set_fact:
         fw_norm: >-
           {{
-            fw_banner_raw
-            if (fw_banner_raw | lower is search('-r[0-9]+$'))
-            else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
+            (fw_banner_raw | lower) is search('-r[0-9]+$')
+            | ternary(
+                fw_banner_raw,
+                (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
+              )
           }}
 
-    - name: Debug normalized firmware (indoor_fwver)
+    - name: Debug show normalized indoor firmware value
       when: fw_norm is defined
       delegate_to: localhost
       ansible.builtin.debug:
-        msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
+        msg: "Normalized indoor_fwver={{ fw_norm }} (from {{ fw_banner_raw }})"
 
-    - name: Publish custom_field_set indoor_fwver
+    - name: Publish custom field indoor_fwver to control queue
       when:
         - fw_norm is defined
         - (fw_norm | length) > 0
@@ -559,8 +641,8 @@
       register: rmq_customfield_fw
       changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
 
-    # MISMATCH path: firmware readable but does NOT match expected target
-    - name: Build mismatch journal payload
+    # MISMATCH path
+    - name: Build journal payload for version mismatch after upgrade
       when: (read_ok | bool) and (not (version_match | bool))
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -568,11 +650,11 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "journal_add"
           task_result: >-
-            afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }})
             Expected='{{ expected_norm }}' Got='{{ banner_raw }}'
             Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
 
-    - name: Publish mismatch journal to control queue
+    - name: Publish journal for version mismatch
       when: journal_mismatch_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -594,12 +676,12 @@
       register: rmq_j_mismatch
       changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
 
-    - name: Stop host after mismatch evaluation
+    - name: Stop host after mismatch path is handled
       when: journal_mismatch_payload is defined
       ansible.builtin.meta: end_host
 
-    # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
-    - name: Build failure journal payload (indoor firmware read failed)
+    # FAILURE and RETRY path
+    - name: Build journal payload for indoor firmware read failure
       when: not (read_ok | bool)
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -607,11 +689,11 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "journal_add"
           task_result: >-
-            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }})
             fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
             Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
 
-    - name: Publish failure journal to control queue
+    - name: Publish journal for indoor firmware read failure
       when: journal_fail_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -621,8 +703,7 @@
         password: "{{ rmq_pass }}"
         force_basic_auth: true
         status_code: 200
-        headers:
-          content-type: "application/json"
+        headers: { content-type: "application/json" }
         body_format: json
         body:
           properties:
@@ -633,14 +714,14 @@
       register: rmq_pub_fail
       changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
 
-    - name: Compute retry parameters (10 minutes)
+    - name: Compute retry parameters wait ten minutes before next attempt
       when: not (read_ok | bool)
       ansible.builtin.set_fact:
         next_attempt: "{{ (attempt | int) + 1 }}"
         next_delay_sec: 600
         next_delay_ms: "{{ 600000 }}"
 
-    - name: Build final gave-up journal (max attempts reached)
+    - name: Build final gave up journal when max attempts reached after read failure
       when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int))
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -648,11 +729,11 @@
           inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
           task_name: "journal_add"
           task_result: >-
-            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
+            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }})
             Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
             Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
 
-    - name: Publish final gave-up journal
+    - name: Publish final gave up journal after read failure
       when: journal_gaveup_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -662,8 +743,7 @@
         password: "{{ rmq_pass }}"
         force_basic_auth: true
         status_code: 200
-        headers:
-          content-type: "application/json"
+        headers: { content-type: "application/json" }
         body_format: json
         body:
           properties:
@@ -674,8 +754,8 @@
       register: rmq_pub_gaveup
       changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
 
-    # Only schedule next attempt if we still have budget left
-    - name: Build delayed payload for next indoor attempt (10 min)
+    # Only schedule next attempt if budget left
+    - name: Build delayed payload for next indoor attempt wait ten minutes
       when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int))
       delegate_to: localhost
       ansible.builtin.set_fact:
@@ -686,10 +766,22 @@
           max_attempts: "{{ effective_max_attempts | int }}"
           correlation_id: "{{ correlation_id | default('') }}"
           original_emitted_at: "{{ original_emitted_at | default('') }}"
+          target_version_full: "{{ target_version_full }}"    
+          target_version: "{{ target_version | default(target_version_full) }}" 
           current_delay_sec: "{{ next_delay_sec | int }}"
           schema_version: 1
 
-    - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
+    - name: Debug | next attempt backoff plan
+      delegate_to: localhost
+      when: debug_aic | bool and (version_match | default(false) | bool) == false
+      ansible.builtin.debug:
+        msg:
+          - "attempt={{ attempt | default('') }} / max_attempts={{ max_attempts | default('') }}"
+          - "next_delay_sec={{ next_delay_sec | default('') }}"
+          - "routing_key={{ routing_key | default('deviceconfig') }}"
+          - "correlation_id={{ correlation_id | default('') }}"
+
+    - name: Publish delayed next indoor attempt to holding with TTL routed to deviceconfig
       when: delayed_payload is defined
       delegate_to: localhost
       ansible.builtin.uri:
@@ -699,8 +791,7 @@
         password: "{{ rmq_pass }}"
         force_basic_auth: true
         status_code: 200
-        headers:
-          content-type: "application/json"
+        headers: { content-type: "application/json" }
         body_format: json
         body:
           properties:
@@ -714,28 +805,28 @@
       changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
 
   post_tasks:
-    - name: Cleanup tunnel and temp IP
+    - name: Cleanup section tunnel and temporary IP
       block:
         - ansible.builtin.debug:
-            msg: "Cleaning up tunnel + temp IP"
+            msg: "Cleaning up tunnel and temporary IP"
           changed_when: false
           delegate_to: localhost
       always:
-        - name: Close tunnel
+        - name: Close SSH ControlMaster tunnel best effort
           delegate_to: localhost
           ansible.builtin.shell: |
             ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
           changed_when: false
           ignore_errors: true
 
-        - name: Remove control dir
+        - name: Remove temporary control directory best effort
           delegate_to: localhost
           ansible.builtin.file:
             path: "{{ _ctrl_dir | default('/tmp/none') }}"
             state: absent
           ignore_errors: true
 
-        - name: Remove temporary IP from DEV1
+        - name: Remove temporary IP from DEV1 best effort
           ansible.builtin.raw: >
             {{ pathprefix }}
             ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}

files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries

@@ -1,201 +0,0 @@
----
-- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
-  hosts: all
-  gather_facts: no
-
-  vars:
-    # ---------------- BusyBox-safe path prefix ----------------
-    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
-
-    # ---------------- DEV1 (outer device) ----------------
-    dev1_user: "root"
-    dev1_pass: "wavewave"
-    dev1_iface: "br-wan"
-
-    # ---------------- DEV2 (indoor behind DEV1) ----------------
-    dev2_host: "192.168.1.1"
-    dev2_port: 22
-    dev2_side_ip: "192.168.1.11/24"
-    dev2_ssh_user: "root"
-    dev2_passfiles:
-      - "basicpass"
-      - "basicpass2"
-
-    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
-    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
-    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
-    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
-    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
-    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
-    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
-    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
-
-  tasks:
-    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
-    - name: Read DEV1 hostname
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
-      register: dev1_host_read
-      changed_when: false
-
-    - name: Stop if DEV1 hostname mismatch
-      ansible.builtin.meta: end_host
-      when: (dev1_host_read.stdout | trim | length > 0) and
-            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
-
-    # ---------------- Step 2: Setup temporary IP for reachability ----------------
-    - name: Add temporary IP on DEV1 (ignore if exists)
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
-      register: add_ip
-      changed_when: add_ip.rc == 0
-      failed_when: >
-        add_ip.rc != 0 and
-        ('File exists' not in (add_ip.stdout | default(''))) and
-        ('File exists' not in (add_ip.stderr | default('')))
-
-    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
-    - name: Pick random free local port
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        for i in $(seq 1 25); do
-          p="$(shuf -i 20000-39999 -n1)"
-          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
-        done
-        exit 1
-      register: pick_port
-      changed_when: false
-
-    - name: Stop if no free local port
-      ansible.builtin.meta: end_host
-      when: (pick_port.stdout | trim | length) == 0
-
-    - name: Record chosen local port and control dir
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _local_port: "{{ pick_port.stdout | trim }}"
-        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
-
-    - name: Build tunnel control socket path
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
-
-
-    - name: Start SSH ControlMaster tunnel via DEV1
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        sshpass -p '{{ dev1_pass }}' ssh -f -N \
-          -M -S "{{ _ctrl_sock }}" \
-          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
-          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
-      args:
-        executable: /bin/bash
-      register: start_tunnel
-      changed_when: true
-
-    # ---------------- Step 4: Determine working password for DEV2 ----------------
-    - name: Try both passfiles for DEV2
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        for f in {{ dev2_passfiles | join(' ') }}; do
-          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
-            echo "$f"; exit 0;
-          fi
-        done
-        echo NONE
-      register: dev2_passfile_try
-      changed_when: false
-
-    - name: Save selected DEV2 passfile
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
-      changed_when: false
-
-    - name: Stop if no valid passfile found
-      ansible.builtin.meta: end_host
-      when: dev2_passfile_used == "NONE"
-
-    # ---------------- Step 5: Read firmware version on DEV2 ----------------
-    - name: Read firmware version from DEV2
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
-          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
-      register: dev2_fwver
-      changed_when: false
-      ignore_errors: true
-
-    - name: Show firmware version readout
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
-
-    # ---------------- Step 6: Publish result journal ----------------
-    - name: Build journal payload
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
-            tunnel_port={{ _local_port }}
-
-    - name: Publish journal to control queue
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_fwver
-      changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool)
-
-  post_tasks:
-    - name: Cleanup tunnel and temp IP
-      block:
-        - ansible.builtin.debug:
-            msg: "Cleaning up tunnel + temp IP"
-          changed_when: false
-          delegate_to: localhost
-      always:
-        - name: Close tunnel
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
-          changed_when: false
-          ignore_errors: true
-
-        - name: Remove control dir
-          delegate_to: localhost
-          ansible.builtin.file:
-            path: "{{ _ctrl_dir | default('/tmp/none') }}"
-            state: absent
-          ignore_errors: true
-
-        - name: Remove temporary IP from DEV1
-          ansible.builtin.raw: >
-            {{ pathprefix }}
-            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
-          register: del_ip
-          failed_when: false
-          changed_when: false

files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors

@@ -1,644 +0,0 @@
----
-- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
-  hosts: all
-  gather_facts: no
-
-  vars:
-    # ---------------- BusyBox-safe path prefix ----------------
-    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
-
-    # ---------------- DEV1 (outer device) ----------------
-    dev1_user: "root"
-    dev1_pass: "wavewave"
-    dev1_iface: "br-wan"
-
-    # ---------------- DEV2 (indoor behind DEV1) ----------------
-    dev2_host: "192.168.1.1"
-    dev2_port: 22
-    dev2_side_ip: "192.168.1.11/24"
-    dev2_ssh_user: "root"
-    dev2_passfiles:
-      - "basicpass"
-      - "basicpass2"
-
-    # ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
-    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
-    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
-    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
-    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
-    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
-    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
-    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
-
-  tasks:
-    # ---- Normalize metadata (from delayed message) ----
-    - name: Normalize after-upgrade metadata
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        attempt: "{{ (attempt | default(1)) | int }}"
-        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
-        correlation_id: "{{ correlation_id | default('') }}"
-        original_emitted_at: "{{ original_emitted_at | default('') }}"
-        target_version: "{{ target_version | default('') }}"
-        target_version_full: "{{ target_version | default('') }}"
-
-    # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
-    - name: Check if TCP/22 on DEV1 is reachable
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
-      register: nc_probe
-      changed_when: false
-      ignore_errors: true
-
-    # ---- If TCP down: journal + schedule next try or give up ----
-    - name: Build failure journal (TCP unreachable) + mark retry
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            TCP 22 unreachable. Correlation={{ correlation_id }}
-            Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-        _needs_retry: true
-
-    - name: Publish failure journal (TCP unreachable)
-      when: nc_probe.rc != 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers: { content-type: "application/json" }
-        body_format: json
-        body:
-          properties: { content_type: "application/json" }
-          routing_key: "{{ control_queue }}"
-          payload: "{{ fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_tcp_fail
-      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
-
-    - name: Compute next-attempt delay (10 minutes) and counters
-      when: (_needs_retry | default(false)) | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        next_attempt: "{{ attempt | int + 1 }}"
-        next_delay_sec: 600
-        next_delay_ms: 600000
-
-    - name: Build final gave-up journal (max attempts reached)
-      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_gaveup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Exhausted attempts. Last error path=TCP.
-            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
-
-    - name: Publish final gave-up journal
-      when: journal_gaveup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers: { content-type: "application/json" }
-        body_format: json
-        body:
-          properties: { content_type: "application/json" }
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_gaveup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_j_gaveup
-      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
-
-    - name: Stop host after final gave-up
-      when: journal_gaveup_payload is defined
-      ansible.builtin.meta: end_host
-
-    - name: Build delayed after-upgrade payload for next attempt (10m)
-      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        delayed_payload:
-          task_name: "afterupgrade_indoor_check"
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          attempt: "{{ next_attempt | int }}"
-          max_attempts: "{{ effective_max_attempts | int }}"
-          correlation_id: "{{ correlation_id }}"
-          original_emitted_at: "{{ original_emitted_at }}"
-          target_version: "{{ target_version_full }}"
-          current_delay_sec: "{{ next_delay_sec | int }}"
-          schema_version: 1
-
-    - name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
-      when: delayed_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers: { content-type: "application/json" }
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-            expiration: "{{ (next_delay_ms | int) | string }}"
-            correlation_id: "{{ correlation_id }}"
-          routing_key: "deviceconfig"
-          payload: "{{ delayed_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_next
-      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
-
-    - name: Stop host after scheduling next attempt
-      when: (_needs_retry | default(false)) | bool
-      ansible.builtin.meta: end_host
-
-
-    # ---------------- Step 1: Read DEV1 hostname and sanity ----------------
-    - name: Read DEV1 hostname
-      when: nc_probe.rc == 0
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
-      register: dev1_host_read
-      changed_when: false
-
-    - name: Stop if DEV1 hostname mismatch
-      ansible.builtin.meta: end_host
-      when: (dev1_host_read.stdout | trim | length > 0) and
-            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
-
-    # ---------------- Step 2: Setup temporary IP for reachability ----------------
-    - name: Add temporary IP on DEV1 (ignore if exists)
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
-      register: add_ip
-      changed_when: add_ip.rc == 0
-      failed_when: >
-        add_ip.rc != 0 and
-        ('File exists' not in (add_ip.stdout | default(''))) and
-        ('File exists' not in (add_ip.stderr | default('')))
-
-    # ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
-    - name: Pick random free local port
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        for i in $(seq 1 25); do
-          p="$(shuf -i 20000-39999 -n1)"
-          if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
-        done
-        exit 1
-      register: pick_port
-      changed_when: false
-
-    - name: Stop if no free local port
-      ansible.builtin.meta: end_host
-      when: (pick_port.stdout | trim | length) == 0
-
-    - name: Record chosen local port and control dir
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _local_port: "{{ pick_port.stdout | trim }}"
-        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
-
-    - name: Build tunnel control socket path
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
-
-    - name: Start SSH ControlMaster tunnel via DEV1
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        sshpass -p '{{ dev1_pass }}' ssh -f -N \
-          -M -S "{{ _ctrl_sock }}" \
-          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
-          "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
-      args:
-        executable: /bin/bash
-      register: start_tunnel
-      changed_when: true
-
-    # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
-    - name: Small delay for tunnel to settle
-      delegate_to: localhost
-      ansible.builtin.wait_for:
-        timeout: 1
-      changed_when: false
-
-    - name: Verify tunnel master running (ssh -O check)
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
-      register: tun_check
-      changed_when: false
-
-    - name: Debug tunnel check
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "tunnel_check.rc={{ tun_check.rc }}"
-          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
-
-    - name: Sanity confirm tunnel TCP reachability to DEV2
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        nc -z -w5 127.0.0.1 "{{ _local_port }}"
-      register: nc_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Debug reachability result
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "nc.rc={{ nc_probe.rc }}"
-          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
-          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
-
-    - name: Stop if tunnel TCP check failed
-      ansible.builtin.meta: end_host
-      when: nc_probe.rc != 0
-
-    # ---------------- Step 4: Determine working password for DEV2 ----------------
-    - name: Try both passfiles for DEV2
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        for f in {{ dev2_passfiles | join(' ') }}; do
-          if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
-            echo "$f"; exit 0;
-          fi
-        done
-        echo NONE
-      register: dev2_passfile_try
-      changed_when: false
-
-    - name: Save selected DEV2 passfile
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
-      changed_when: false
-
-    - name: Debug selected DEV2 passfile
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg: "dev2_passfile_used={{ dev2_passfile_used }}"
-
-    - name: Stop if no valid passfile found
-      ansible.builtin.meta: end_host
-      when: dev2_passfile_used == "NONE"
-
-    # ---------------- Step 5: Read firmware version on DEV2 ----------------
-    - name: Read firmware version from DEV2
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
-          root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
-      register: dev2_fwver
-      changed_when: false
-      ignore_errors: true
-
-    - name: Show firmware version readout
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
-
-    # ---------------- Retry metadata + success evaluation ----------------
-    - name: Normalize retry metadata for indoor checker
-      ansible.builtin.set_fact:
-        attempt: "{{ (attempt | default(1)) | int }}"
-        effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
-        correlation_id: "{{ correlation_id | default('') }}"
-        original_emitted_at: "{{ original_emitted_at | default('') }}"
-
-    - name: Evaluate indoor success (firmware read OK)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        indoor_success: "{{ (dev2_fwver.rc | default(1) == 0)
-                            and ((dev2_fwver.stdout | default('') | trim) | length > 0)
-                            and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
-
-    # ---------------- Step 6: Publish result journal (+ tags on success) ----------------
-    # SUCCESS PATH: add tag, remove scheduled tag, and journal with "success"
-    - name: Build success journal payload
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_success_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
-            tunnel_port={{ _local_port }}
-            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
-
-    - name: Publish success journal to control queue
-      when: journal_success_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_success_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_success
-      changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
-
-    - name: Build payload to add indoor-update-success tag (DEV1)
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        tag_add_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "tag_add"
-          task_result: "indoor-update-success"
-
-    - name: Publish indoor-update-success tag
-      when: tag_add_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ tag_add_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_tag_add_success
-      changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
-
-    - name: Build payload to remove indoor-restart-scheduled tag (DEV1)
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        tag_remove_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "tag_remove"
-          task_result: "indoor-restart-scheduled"
-
-    - name: Publish indoor-restart-scheduled tag removal
-      when: tag_remove_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ tag_remove_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_tag_remove_sched
-      changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
-
-    # --- Normalize firmware string and set custom field on success ---
-    - name: Capture raw firmware banner from DEV2 (for normalization)
-      when: indoor_success | bool
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}"
-
-    - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
-      when:
-        - indoor_success | bool
-        - (fw_banner_raw | default('') | length) > 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        fw_norm: >-
-          {{
-            fw_banner_raw
-            if (fw_banner_raw | lower is search('-r[0-9]+$'))
-            else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
-          }}
-
-    - name: Debug normalized firmware (indoor_fwver)
-      when: fw_norm is defined
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
-
-    - name: Publish custom_field_set indoor_fwver
-      when:
-        - indoor_success | bool
-        - fw_norm is defined
-        - (fw_norm | length) > 0
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ {
-            'inscope_device': (ansible_hostname | default(inventory_hostname)),
-            'task_name': 'custom_field_set',
-            'task_add1': 'indoor_fwver',
-            'task_result': fw_norm
-          } | to_json }}"
-          payload_encoding: "string"
-      register: rmq_customfield_fw
-      changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
-
-    # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
-    - name: Build failure journal payload (indoor firmware read failed)
-      when: not (indoor_success | bool)
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_fail_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
-            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
-
-    - name: Publish failure journal to control queue
-      when: journal_fail_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_fail_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_fail
-      changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
-
-    - name: Compute retry parameters (10 minutes)
-      when: not (indoor_success | bool)
-      ansible.builtin.set_fact:
-        next_attempt: "{{ (attempt | int) + 1 }}"
-        next_delay_sec: 600
-        next_delay_ms: "{{ 600000 }}"
-
-    - name: Build final gave-up journal (max attempts reached)
-      when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int))
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        journal_gaveup_payload:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
-            Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
-            Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
-
-    - name: Publish final gave-up journal
-      when: journal_gaveup_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_gaveup_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_gaveup
-      changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
-
-    # Only schedule next attempt if we still have budget left
-    - name: Build delayed payload for next indoor attempt (10 min)
-      when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int))
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        delayed_payload:
-          task_name: "afterupgrade_indoor_check"
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          attempt: "{{ next_attempt | int }}"
-          max_attempts: "{{ effective_max_attempts | int }}"
-          correlation_id: "{{ correlation_id | default('') }}"
-          original_emitted_at: "{{ original_emitted_at | default('') }}"
-          current_delay_sec: "{{ next_delay_sec | int }}"
-          schema_version: 1
-
-    - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
-      when: delayed_payload is defined
-      delegate_to: localhost
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-            expiration: "{{ (next_delay_ms | int) | string }}"
-            correlation_id: "{{ correlation_id | default('') }}"
-          routing_key: "deviceconfig"
-          payload: "{{ delayed_payload | to_json }}"
-          payload_encoding: "string"
-      register: rmq_pub_next
-      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
-
-  post_tasks:
-    - name: Cleanup tunnel and temp IP
-      block:
-        - ansible.builtin.debug:
-            msg: "Cleaning up tunnel + temp IP"
-          changed_when: false
-          delegate_to: localhost
-      always:
-        - name: Close tunnel
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
-          changed_when: false
-          ignore_errors: true
-
-        - name: Remove control dir
-          delegate_to: localhost
-          ansible.builtin.file:
-            path: "{{ _ctrl_dir | default('/tmp/none') }}"
-            state: absent
-          ignore_errors: true
-
-        - name: Remove temporary IP from DEV1
-          ansible.builtin.raw: >
-            {{ pathprefix }}
-            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
-          register: del_ip
-          failed_when: false
-          changed_when: false
-

files/ansible-playbooks/update-indoor.yml-bckp1

@@ -1,997 +0,0 @@
----
-# update-secondline.yml
-- name: Second-line upgrade tunneled noninvasive engine
-  hosts: all
-  gather_facts: no
-
-  vars:
-    # Busybox-safe PATH prefix for all remote raw calls on DEV1
-    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
-
-    # DEV1 credentials (stable, like rebootin222)
-    dev1_user: "root"
-    dev1_pass: "wavewave"
-
-    # Tunnel target DEV2 behind DEV1
-    dev2_host: "192.168.1.1"
-    dev2_port: 22
-
-    # Temp IP we add to DEV1 so it can reach DEV2
-    dev2_side_ip: "192.168.1.11/24"
-    dev1_iface: "br-wan"
-
-    # DEV2 behind the tunnel
-    dev2_ssh_user: "root"
-    dev2_passfiles:
-      - "basicpass"
-      - "basicpass2"
-
-    # SSH options used from controller
-    ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
-
-    # Image to stage on DEV2 noninvasive we do not run update -w here
-    image_filename: "fox200-2.2.1-r6801.bin"
-    image_md5: "56b7211709de617e058b98d4204e2562"
-    # Optional SHA256; leave empty to skip SHA256 checks
-    image_sha256: ""
-    dev2_image_dir: "/tmp"
-    dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
-
-    # Reboot delay in minutes integer string if set we will schedule reboot on DEV2
-    rebootin: ""
-
-    # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
-    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
-    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
-    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
-    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
-    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
-    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
-    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
-
-  pre_tasks:
-    # ------------------------------- Hostname sanity DEV1 -------------------------------
-    - name: Read DEV1 hostname from env busybox safe
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        (cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
-      register: dev1_host_read
-      changed_when: false
-
-    - name: Stop if DEV1 connected hostname differs from inventory
-      ansible.builtin.meta: end_host
-      when: (dev1_host_read.stdout | trim | length > 0) and
-            ((dev1_host_read.stdout | trim) != (inventory_hostname | string))
-
-  tasks:
-    # ---------------------------- Idempotent temp IP on DEV1 ----------------------------
-    - name: Add temporary IP on DEV1 idempotent treat File exists as OK
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
-      register: add_ip
-      changed_when: add_ip.rc == 0
-      failed_when: >
-        add_ip.rc != 0
-        and ('File exists' not in (add_ip.stdout | default('')))
-        and ('File exists' not in (add_ip.stderr | default('')))
-
-    - name: Debug add ip results
-      ansible.builtin.debug:
-        msg:
-          - "add_ip.rc={{ add_ip.rc | default('') }}"
-          - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
-          - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
-
-    # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ----------------------------
-    - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
-      register: dev2_mac_scan
-      changed_when: false
-
-    - name: Set dev2_mac from fdb scan (if any)
-      ansible.builtin.set_fact:
-        dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
-      changed_when: false
-
-    - name: Remove current ARP entry for DEV2 on DEV1 (best-effort)
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
-      register: dev2_arp_del
-      changed_when: false
-      failed_when: false
-
-    - name: Add static ARP for DEV2 on DEV1
-      ansible.builtin.raw: >
-        {{ pathprefix }}
-        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
-      register: dev2_arp_add
-      changed_when: dev2_arp_add.rc == 0
-      failed_when: >
-        dev2_arp_add.rc != 0
-        and ('File exists' not in (dev2_arp_add.stdout | default('')))
-        and ('File exists' not in (dev2_arp_add.stderr | default('')))
-
-    - name: Debug static ARP result
-      ansible.builtin.debug:
-        msg:
-          - "dev2_mac={{ dev2_mac | default('UNSET') }}"
-          - "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
-          - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
-          - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
-
-    - name: Skip ARP add because MAC not found
-      when: dev2_mac is not defined or dev2_mac | length == 0
-      ansible.builtin.debug:
-        msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
-
-    # ---------------------------- Local tunnel preparation ----------------------------
-    - name: Pick a free local port for the tunnel controller
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        pick() {
-            for i in $(seq 1 25); do
-                p="$(shuf -i 20000-39999 -n 1)"
-                if command -v ss >/dev/null 2>&1; then
-                    if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
-                        echo "$p"; return 0
-                    fi
-                else
-                    if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
-                        echo "$p"; return 0
-                    fi
-                fi
-            done
-            return 1
-        }
-        pick
-      register: pick_port
-      changed_when: false
-
-    - name: Stop if no free local port
-      ansible.builtin.meta: end_host
-      when: (pick_port.stdout | trim | length) == 0
-
-    - name: Record chosen local port and control directory
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _local_port: "{{ pick_port.stdout | trim }}"
-        _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
-
-    - name: Build tunnel control socket path
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
-
-    - name: Debug chosen tunnel params
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "local_port={{ _local_port }}"
-          - "ctrl_dir={{ _ctrl_dir }}"
-          - "ctrl_sock={{ _ctrl_sock }}"
-
-    - name: issue arping with own IP
-      ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
-
-    # ---------------------------- Start SSH local forward via DEV1 ----------------------------
-    - name: Start SSH tunnel via DEV1 background ControlMaster
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        USER="{{ dev1_user }}"
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
-            -M -S "{{ _ctrl_sock }}" \
-            -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
-            "${USER}@${HOST}"
-      args:
-        executable: /bin/bash
-      register: start_tunnel
-      changed_when: true
-
-    - name: Small delay for tunnel to settle
-      delegate_to: localhost
-      ansible.builtin.wait_for:
-        timeout: 1
-      changed_when: false
-
-    - name: Verify tunnel master running proper O check with destination
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        HOST="{{ ansible_host | default(inventory_hostname) }}"
-        ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
-      register: tun_check
-      changed_when: false
-
-    - name: Debug tunnel check
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "tunnel_check.rc={{ tun_check.rc }}"
-          - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
-
-    # ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
-    - name: Sanity confirm tunnel TCP reachability to DEV2
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        nc -z -w5 127.0.0.1 "{{ _local_port }}"
-      register: nc_probe
-      changed_when: false
-      ignore_errors: true
-
-    - name: Debug reachability result
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "nc.rc={{ nc_probe.rc }}"
-          - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
-          - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
-
-    - name: Stop if tunnel TCP check failed
-      ansible.builtin.meta: end_host
-      when: nc_probe.rc != 0
-
-    - name: Sanity show presence and permissions of passfiles on controller
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
-      register: dev2_ls
-      changed_when: false
-
-    - name: Debug passfiles listing
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "{{ (dev2_ls.stdout | default('')) | trim }}"
-          - "{{ (dev2_ls.stderr | default('')) | trim }}"
-
-    # ---------------------------- Pick DEV2 password root only ----------------------------
-    - name: Try DEV2 login with basicpass root
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f basicpass ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
-      register: dev2_try_basicpass
-      changed_when: false
-      ignore_errors: true
-
-    - name: Select basicpass as working passfile
-      when: dev2_try_basicpass.rc == 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "basicpass"
-      changed_when: false
-
-    - name: Try DEV2 login with basicpass2 root only if first failed
-      when: dev2_passfile_used is not defined
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f basicpass2 ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
-      register: dev2_try_basicpass2
-      changed_when: false
-      ignore_errors: true
-
-    - name: Select basicpass2 as working passfile
-      when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "basicpass2"
-      changed_when: false
-
-    - name: Set dev2_passfile_used to NONE if neither worked
-      when: dev2_passfile_used is not defined
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        dev2_passfile_used: "NONE"
-      changed_when: false
-
-    - name: Debug selected DEV2 passfile or NONE
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "dev2_passfile_used={{ dev2_passfile_used }}"
-          - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}"
-          - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}"
-
-    - name: Read DEV2 hostname via tunnel (busybox-safe)
-      when: dev2_passfile_used != "NONE"
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
-      args:
-        executable: /bin/bash
-      register: dev2_host_read
-      changed_when: false
-
-    - name: Normalize hostnames for strict compare
-      ansible.builtin.set_fact:
-        _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
-        _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
-        _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
-
-    - name: Debug hostname bytes (hex)
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "inv={{ _inv_hn | tojson }}"
-          - "dev1={{ _dev1_hn | tojson }}"
-          - "dev2={{ _dev2_hn | tojson }}"
-
-    # ---------------- Hostname equality guard (soft-journal and stop) ----------------
-    - name: Enforce DEV2 hostname equals inventory and DEV1
-      block:
-        - name: Fail if DEV2 hostname differs from inventory/DEV1
-          ansible.builtin.fail:
-            msg: >
-              Hostname mismatch: DEV2='{{ _dev2_hn }}',
-              inventory='{{ _inv_hn }}',
-              DEV1='{{ _dev1_hn }}'
-          when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
-      rescue:
-        - name: Initialize journal array for hostname mismatch
-          ansible.builtin.set_fact:
-            _journal: []
-            _blocked: true
-            _prep_blocked: false
-          delegate_to: localhost
-
-        - name: Append hostname mismatch to journal
-          ansible.builtin.set_fact:
-            _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}"
-          delegate_to: localhost
-
-        - name: Build control queue payload for indoor aborted journal (hostname)
-          ansible.builtin.set_fact:
-            journal_indoor_aborted:
-              inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-              task_name: "journal_add"
-              task_result: >-
-                indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
-          delegate_to: localhost
-
-        - name: Publish indoor aborted journal to control queue (hostname)
-          ansible.builtin.uri:
-            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-            method: POST
-            user: "{{ rmq_user }}"
-            password: "{{ rmq_pass }}"
-            force_basic_auth: true
-            status_code: 200
-            headers:
-              content-type: "application/json"
-            body_format: json
-            body:
-              properties:
-                content_type: "application/json"
-              routing_key: "{{ control_queue }}"
-              payload: "{{ journal_indoor_aborted | to_json }}"
-              payload_encoding: "string"
-          register: rmq_journal_indoor_aborted_hn_resp
-          changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
-          failed_when: >
-            (rmq_journal_indoor_aborted_hn_resp.status != 200) or
-            (rmq_journal_indoor_aborted_hn_resp.json is not defined) or
-            (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
-          delegate_to: localhost
-
-        - name: Stop host after hostname mismatch
-          ansible.builtin.meta: end_host
-
-    # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
-    - name: Initialize journal and flags
-      ansible.builtin.set_fact:
-        _journal: []
-        _prep_blocked: false
-        _blocked: false
-
-    - name: Build specific marker filename (no .bin)
-      ansible.builtin.set_fact:
-        _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}"
-
-    - name: Check for any preparation markers on DEV2 (wildcard count)
-      when: dev2_passfile_used != "NONE"
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=10 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "ls /tmp/prepared_for* 2>/dev/null | wc -l"
-      args:
-        executable: /bin/bash
-      register: dev2_prep_count
-      changed_when: false
-      ignore_errors: true
-
-    - name: Soft-block if any prep markers already present
-      when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
-      ansible.builtin.set_fact:
-        _prep_blocked: true
-        _blocked: true
-        _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
-
-    # ---------------------------- DEV2 version firmux primary check ----------------------------
-    - name: Read DEV2 firmux release
-      when: dev2_passfile_used != "NONE"
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "cat /usr/lib/release/firmux 2>/dev/null || true"
-      args:
-        executable: /bin/bash
-      register: dev2_firmux
-      changed_when: false
-      ignore_errors: true
-
-    - name: Debug firmux content DEV2
-      when: dev2_firmux is defined
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
-
-    # ---------------------------- Normalize rebootin early ----------------------------
-    - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)"
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}"
-        _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}"
-        _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}"
-        _reboot_minutes: >-
-          {{
-            0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'])
-            else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1)
-          }}
-
-    - name: "Normalize rebootin (phase 2: requested flag)"
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}"
-
-    # ---------------------------- Journal: indoor start (after we know we can proceed) ----------------------------
-    - name: Build control queue payload for indoor start journal
-      ansible.builtin.set_fact:
-        journal_indoor_start:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            Indoor: Dev2 is reachable, starting with update.
-            image={{ image_filename }}, md5={{ image_md5 }},
-            firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }},
-            reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }}
-      when: dev2_passfile_used != "NONE"
-      delegate_to: localhost
-
-    - name: Publish indoor start journal to control queue
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_indoor_start | to_json }}"
-          payload_encoding: "string"
-      register: rmq_journal_indoor_start_resp
-      changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
-      failed_when: >
-        (rmq_journal_indoor_start_resp.status != 200) or
-        (rmq_journal_indoor_start_resp.json is not defined) or
-        (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
-      when: journal_indoor_start is defined
-      delegate_to: localhost
-
-    # ---------------------------- Stage image on DEV2 no write ----------------------------
-    - name: Check local presence of image file
-      delegate_to: localhost
-      ansible.builtin.stat:
-        path: "{{ image_filename }}"
-      register: local_img
-
-    - name: Soft-block if image missing locally
-      when: not local_img.stat.exists
-      ansible.builtin.set_fact:
-        _blocked: true
-        _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
-
-    - name: Compute local md5 of the image
-      when: local_img.stat.exists
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        md5sum "{{ image_filename }}" | awk '{print $1}'
-      register: local_md5
-      changed_when: false
-
-    - name: Verify local md5 matches expected
-      when: local_img.stat.exists
-      delegate_to: localhost
-      ansible.builtin.assert:
-        that:
-          - (local_md5.stdout | trim) == image_md5
-        fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
-        success_msg: "Local md5 matches expected"
-
-    # Optional SHA256 local
-    - name: Compute local sha256 of the image (if provided)
-      when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        sha256sum "{{ image_filename }}" | awk '{print $1}'
-      register: local_sha256
-      changed_when: false
-      ignore_errors: true
-
-    - name: Soft-block if local sha256 mismatch or unavailable
-      when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
-      ansible.builtin.set_fact:
-        _prep_blocked: true
-        _blocked: true
-        _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
-
-    # fw_printenv health before upload (soft-fail)
-    - name: Read fw_printenv size (line count) on DEV2
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=10 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "(fw_printenv 2>/dev/null | wc -l) || echo 0"
-      args:
-        executable: /bin/bash
-      register: dev2_fwenv_wc
-      changed_when: false
-      ignore_errors: true
-
-    - name: Soft-block if fw_printenv too small
-      when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
-      ansible.builtin.set_fact:
-        _prep_blocked: true
-        _blocked: true
-        _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
-
-    - name: Check existing image md5 on DEV2 NOFILE if absent
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
-      args:
-        executable: /bin/bash
-      register: dev2_md5_before
-      changed_when: false
-
-    - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" scp \
-          -P "$PORT" \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/"
-      args:
-        executable: /bin/bash
-      register: scp_push
-      changed_when: true
-
-    - name: Compute md5 of image on DEV2 after copy or if exists
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
-      args:
-        executable: /bin/bash
-      register: dev2_md5_after
-      changed_when: false
-
-    - name: Verify DEV2 md5 matches expected (soft-fail journal)
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
-      ansible.builtin.set_fact:
-        _prep_blocked: true
-        _blocked: true
-        _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
-
-    # Optional SHA256 remote
-    - name: Compute sha256 of image on DEV2 (if provided)
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=10 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
-      args:
-        executable: /bin/bash
-      register: dev2_sha256_after
-      changed_when: false
-      ignore_errors: true
-
-    - name: Soft-block if DEV2 sha256 mismatch/unavailable
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
-      ansible.builtin.set_fact:
-        _prep_blocked: true
-        _blocked: true
-        _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
-
-    - name: Run noninvasive update check on DEV2 update c
-      when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=15 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "update -c '{{ dev2_image_path }}' 2>&1 || true"
-      args:
-        executable: /bin/bash
-      register: dev2_update_check
-      changed_when: false
-
-    - name: Debug update c output DEV2
-      when: dev2_update_check is defined
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
-
-    - name: Soft-block if update -c did not say valid
-      when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
-      ansible.builtin.set_fact:
-        _prep_blocked: true
-        _blocked: true
-        _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
-
-    # ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
-    - name: Build control queue payload for indoor aborted journal
-      ansible.builtin.set_fact:
-        journal_indoor_aborted:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
-      when: (_blocked | default(false))
-      delegate_to: localhost
-
-    - name: Publish indoor aborted journal to control queue
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_indoor_aborted | to_json }}"
-          payload_encoding: "string"
-      register: rmq_journal_indoor_aborted_resp
-      changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
-      failed_when: >
-        (rmq_journal_indoor_aborted_resp.status != 200) or
-        (rmq_journal_indoor_aborted_resp.json is not defined) or
-        (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
-      when: journal_indoor_aborted is defined
-      delegate_to: localhost
-
-    # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
-    - name: Upgrade write and flip block
-      when: not (_blocked | default(false))
-      block:
-        - name: Write image on DEV2 (this will take a while)
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            set -e
-            PORT="{{ _local_port }}"
-            sshpass -f "{{ dev2_passfile_used }}" ssh \
-              -o AddressFamily=inet \
-              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-              -o ConnectTimeout=0 \
-              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-              "update -w '{{ dev2_image_path }}'"
-          args:
-            executable: /bin/bash
-          register: dev2_up_write
-          changed_when: true
-          failed_when: dev2_up_write.stdout is not search('update is complete')
-
-        - name: Read current active partition on DEV2
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            set -e
-            PORT="{{ _local_port }}"
-            sshpass -f "{{ dev2_passfile_used }}" ssh \
-              -o AddressFamily=inet \
-              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-              -o ConnectTimeout=10 \
-              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-              "fw_printenv active | awk -F= '/^active=/{print \$2}'"
-          args:
-            executable: /bin/bash
-          register: dev2_active_before
-          changed_when: false
-          failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
-
-        - name: Determine new active value for DEV2
-          when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
-          ansible.builtin.set_fact:
-            dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
-
-        - name: Switch active partition to {{ dev2_new_active }}
-          when: dev2_new_active is defined
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            set -e
-            PORT="{{ _local_port }}"
-            sshpass -f "{{ dev2_passfile_used }}" ssh \
-              -o AddressFamily=inet \
-              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-              -o ConnectTimeout=10 \
-              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-              "fw_setenv active {{ dev2_new_active }}"
-          args:
-            executable: /bin/bash
-          register: dev2_setenv_out
-          changed_when: true
-
-        - name: Verify active partition flipped on DEV2
-          when: dev2_new_active is defined
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            set -e
-            PORT="{{ _local_port }}"
-            sshpass -f "{{ dev2_passfile_used }}" ssh \
-              -o AddressFamily=inet \
-              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-              -o ConnectTimeout=10 \
-              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-              "fw_printenv active | awk -F= '/^active=/{print \$2}'"
-          args:
-            executable: /bin/bash
-          register: dev2_active_after
-          changed_when: false
-          failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
-
-        - name: Create specific prep marker on DEV2 (no generic)
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            set -e
-            PORT="{{ _local_port }}"
-            sshpass -f "{{ dev2_passfile_used }}" ssh \
-              -o AddressFamily=inet \
-              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-              -o ConnectTimeout=10 \
-              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-              "touch '{{ _marker_specific }}'"
-          args:
-            executable: /bin/bash
-          register: dev2_marker_write
-          changed_when: true
-          ignore_errors: true
-
-    # ---------------------------- Reboot scheduling (normalized) ----------------------------
-    - name: Schedule delayed reboot on DEV2 minutes
-      when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        SECS="$(( {{ _reboot_minutes | int }} * 60 ))"
-        CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=10 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}"
-      args:
-        executable: /bin/bash
-      register: dev2_reboot_sched
-      changed_when: true
-      ignore_errors: true
-
-    - name: Note reboot request bad format
-      when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
-      ansible.builtin.debug:
-        msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"
-
-    # ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
-    - name: Build control queue payload for indoor updated journal
-      ansible.builtin.set_fact:
-        _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}"
-      when: not (_blocked | default(false))
-      delegate_to: localhost
-
-    - name: Build indoor updated journal payload text
-      ansible.builtin.set_fact:
-        journal_indoor_updated:
-          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-          task_name: "journal_add"
-          task_result: >-
-            Indoor: Dev2 updated, scheduling reboot {{
-              'now' if (_reboot_requested and (_reboot_minutes | int) == 0)
-              else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0)
-              else 'not requested'
-            }}.
-            write_done={{ _write_success }},
-            active_before={{ (dev2_active_before.stdout | default('NA')) | trim }},
-            active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}
-      when: not (_blocked | default(false)) and (_write_success | bool)
-      delegate_to: localhost
-
-    - name: Publish indoor updated journal to control queue
-      ansible.builtin.uri:
-        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-        method: POST
-        user: "{{ rmq_user }}"
-        password: "{{ rmq_pass }}"
-        force_basic_auth: true
-        status_code: 200
-        headers:
-          content-type: "application/json"
-        body_format: json
-        body:
-          properties:
-            content_type: "application/json"
-          routing_key: "{{ control_queue }}"
-          payload: "{{ journal_indoor_updated | to_json }}"
-          payload_encoding: "string"
-      register: rmq_journal_indoor_updated_resp
-      changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
-      failed_when: >
-        (rmq_journal_indoor_updated_resp.status != 200) or
-        (rmq_journal_indoor_updated_resp.json is not defined) or
-        (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
-      when: journal_indoor_updated is defined
-      delegate_to: localhost
-
-    # ---------------------------- summary ----------------------------
-    - name: Summary show key results
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "dev2_passfile_used={{ dev2_passfile_used }}"
-          - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
-          - "local_image_present={{ local_img.stat.exists | default(false) }}"
-          - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
-          - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
-          - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
-          - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}"
-          - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
-          - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
-          - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
-          - "reboot_requested={{ _reboot_requested | default(false) }}"
-          - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}"
-          - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}"
-          - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
-          - "prep_blocked={{ _prep_blocked | default(false) }}"
-          - "blocked={{ _blocked | default(false) }}"
-          - "journal={{ (_journal | default([])) | join(' || ') }}"
-
-  post_tasks:
-    - name: Cleanup always
-      block:
-        - ansible.builtin.debug:
-            msg: "entering cleanup block"
-          changed_when: false
-          delegate_to: localhost
-      always:
-        - name: Close tunnel best effort
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
-          changed_when: false
-          ignore_errors: true
-
-        - name: Remove control dir best effort
-          delegate_to: localhost
-          ansible.builtin.file:
-            path: "{{ _ctrl_dir | default('/tmp/none') }}"
-            state: absent
-          ignore_errors: true
-
-        - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK
-          ansible.builtin.raw: >
-            {{ pathprefix }}
-            ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
-          register: del_ip
-          changed_when: del_ip.rc == 0
-          failed_when: >
-            del_ip.rc != 0
-            and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
-            and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
-
-    - name: Debug del ip results
-      ansible.builtin.debug:
-        msg:
-          - "del_ip.rc={{ del_ip.rc | default('') }}"
-          - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
-          - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
-      when: del_ip is defined

files/ansible-playbooks/wifidebug17.yml-old

@@ -1,441 +0,0 @@
----
-- name: Deploy WiFi debug (single device, linear)
-  hosts: all
-  gather_facts: no
-
-  vars:
-    remote_syslog_ip: "102.38.125.161"
-
-    # Program version (0)
-    wdbg_version: "v17"
-
-    ssh_user: "{{ ansible_user | default('root') }}"
-    ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
-
-    # RabbitMQ (use controls exchange + queue_controls like the reference)  (e)
-    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
-    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
-    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
-    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
-    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
-    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
-    control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
-
-  tasks:
-
-    - block:
-
-        # --- SSH reachability check ---
-        - name: Check SSH connectivity (raw ping)
-          raw: "echo ping"
-          register: ping_result
-          ignore_errors: true
-
-        - block:
-
-############ step 1
-
-            - name: Check if crontab launch is already present in sysstart.lua
-              raw: |
-                grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
-              register: sysstart_cron_check
-              changed_when: false
-              failed_when: false
-
-            - name: Announce presence and skip edits
-              when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
-              debug:
-                msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
-
-            - name: Fetch sysstart.lua via scp (password auth)
-              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
-              delegate_to: localhost
-              command: >
-                sshpass -p {{ ssh_pass | quote }}
-                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
-                /tmp/{{ inventory_hostname }}_sysstart.lua
-              register: scp_get
-              retries: 3
-              delay: 2
-              until: scp_get.rc == 0
-
-            - name: Edit the file locally
-              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
-              delegate_to: localhost
-              lineinfile:
-                path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
-                line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
-                insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
-
-            - name: Push the file back (write to -2)
-              when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
-              delegate_to: localhost
-              command: >
-                sshpass -p {{ ssh_pass | quote }}
-                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-                /tmp/{{ inventory_hostname }}_sysstart.lua
-                {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
-              register: scp_put
-              retries: 3
-              delay: 2
-              until: scp_put.rc == 0
-
-############ step 2
-
-            - name: Compute MD5 of local wifidebug.sh
-              delegate_to: localhost
-              command: md5sum files/wifidebug.sh
-              register: md5_local_wdbg
-              changed_when: false
-
-            - name: Compute MD5 of remote /root/wifidebug.sh
-              raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
-              register: md5_remote_wdbg
-              changed_when: false
-              failed_when: false
-
-            - name: Decide if wifidebug.sh needs upload
-              set_fact:
-                upload_wdbg: >-
-                  {{ (md5_remote_wdbg.rc != 0)
-                     or ((md5_local_wdbg.stdout.split()[0])
-                         != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
-
-            - name: Upload wifidebug.sh via scp (overwrite if changed)
-              when: upload_wdbg | bool
-              delegate_to: localhost
-              command: >
-                sshpass -p {{ ssh_pass | quote }}
-                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-                files/wifidebug.sh
-                {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
-              register: scp_wifidebug
-              retries: 3
-              delay: 2
-              until: scp_wifidebug.rc == 0
-
-            - name: Ensure /root/wifidebug.sh is executable and owned by root
-              raw: |
-                chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
-
-############ step 3
-
-            - name: Ensure /etc/crontabs/root exists (touch with perms)
-              raw: |
-                if [ ! -f /etc/crontabs/root ]; then
-                    touch /etc/crontabs/root
-                fi
-                chown root:root /etc/crontabs/root
-                chmod 0644 /etc/crontabs/root
-
-            - name: Check if cron line already present
-              raw: |
-                grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
-              register: cron_grep
-              failed_when: false
-              changed_when: false
-
-            - name: Upload snippet crond-root to /tmp (only if missing)
-              when: cron_grep.rc != 0
-              delegate_to: localhost
-              command: >
-                sshpass -p {{ ssh_pass | quote }}
-                scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-                files/crond-root
-                {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
-
-            - name: Append snippet to /etc/crontabs/root (only if missing)
-              when: cron_grep.rc != 0
-              raw: |
-                cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
-              register: cron_append
-              changed_when: true
-
-############ step 4
-
-            - name: Skip config.json edit if remote_syslog already matches
-              raw: |
-                EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
-                SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server'  | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
-                if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
-                  echo "SKIP"
-                else
-                  echo "EDIT"
-                fi
-              register: remote_syslog_check
-              changed_when: false
-
-            - name: Set do_edit flag from skip check
-              set_fact:
-                do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
-
-            - name: Fetch /tmp/config.json to controller
-              when: do_edit
-              delegate_to: localhost
-              command: >
-                sshpass -p {{ ssh_pass | quote }}
-                scp -q -o StrictHostKeyChecking=no
-                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
-                /tmp/{{ inventory_hostname }}_config.json
-
-            - name: Strict in-place style-preserving edit (services.remote_syslog only)
-              when: do_edit
-              delegate_to: localhost
-              shell: |
-                awk '
-                  BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
-                  {
-                    line = $0
-                    if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
-                    if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
-                    if (in_rs) {
-                      if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
-                        line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
-                        done_enabled=1
-                      }
-                      if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
-                        line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
-                        done_server=1
-                      }
-                    }
-                    print line
-                    if (in_rs) {
-                      opens = gsub(/{/, "{", $0)
-                      closes = gsub(/}/, "}", $0)
-                      depth += (opens - closes)
-                      if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
-                    }
-                  }
-                ' "/tmp/{{ inventory_hostname }}_config.json" \
-                > "/tmp/{{ inventory_hostname }}_config.json.new"
-              args:
-                executable: /bin/bash
-
-            - name: Push edited config.json.new to remote temp
-              when: do_edit
-              delegate_to: localhost
-              shell: |
-                sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
-                "/tmp/{{ inventory_hostname }}_config.json.new" \
-                {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
-              args:
-                executable: /bin/bash
-
-            - name: Compute MD5 of local .new
-              when: do_edit
-              delegate_to: localhost
-              command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
-              register: md5_local
-              changed_when: false
-
-            - name: Compute MD5 of remote .new (no Python)
-              when: do_edit
-              raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
-              register: md5_remote
-              changed_when: false
-
-            - name: Fail if MD5 mismatch
-              when:
-                - do_edit
-                - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
-              fail:
-                msg: "MD5 mismatch between controller and remote copy — aborting replace."
-
-            - name: Commit new config.json (overwrite original)
-              when: do_edit
-              raw: |
-                cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
-                mv /tmp/config.json.new /tmp/config.json
-
-            - name: Apply config to runtime (sysconf -w)
-              when: do_edit
-              raw: |
-                sysconf -w || /usr/sbin/sysconf -w
-              register: sysconf_result
-              changed_when: true
-              failed_when: sysconf_result.rc not in [0]
-
-            # (d) Pre-restart notice to control queue
-            - name: Build pre-restart journal (explain why)
-              when: do_edit
-              delegate_to: localhost
-              set_fact:
-                prereboot_payload:
-                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
-                  task_name: "journal_add"
-                  task_result: >-
-                    wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
-
-            - name: Publish pre-restart journal to control queue
-              when: do_edit
-              delegate_to: localhost
-              uri:
-                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-                method: POST
-                user: "{{ rmq_user }}"
-                password: "{{ rmq_pass }}"
-                force_basic_auth: true
-                status_code: 200
-                headers:
-                  content-type: "application/json"
-                body_format: json
-                body:
-                  properties:
-                    content_type: "application/json"
-                  routing_key: "{{ control_queue }}"
-                  payload: "{{ prereboot_payload | to_json }}"
-                  payload_encoding: "string"
-              register: rmq_preboot
-              changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
-
-            # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
-
-            - name: Trigger full restart (system-stop; system-start) from controller with 10s cap
-              when: do_edit
-              delegate_to: localhost
-              shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
-              args:
-                executable: /bin/bash
-              register: restart_kick
-              changed_when: true
-              failed_when: false
-
-
-            # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
-            - name: Probe SSH with nc (24 tries, 5s each)
-              when: do_edit
-              delegate_to: localhost
-              shell: "nc -z -w 3 {{ ansible_host }} 22"
-              register: nc_probe
-              retries: 24
-              delay: 5
-              until: nc_probe.rc == 0
-              changed_when: false
-              failed_when: false
-
-            - name: Set ssh_up fact (from nc probe)
-              when: do_edit
-              delegate_to: localhost
-              set_fact:
-                ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
-
-            - name: Fail if SSH did not return after restart
-              when:
-                - do_edit
-                - not ssh_up | bool
-              fail:
-                msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
-
-            # ======== END OF SURGICAL CHANGE ========
-
-            - name: Set result status (success deployed or no change)
-              set_fact:
-                result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
-
-          when: ping_result is succeeded
-
-        - name: Set status fact (no ssh)
-          when: ping_result is failed
-          set_fact:
-            result_status: "NO_SSH"
-
-      rescue:
-        - name: Mark result as failed
-          set_fact:
-            result_status: "FAILED during {{ ansible_failed_task.name }}"
-
-      always:
-
-        - name: Compute inscope device
-          set_fact:
-            inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
-
-        # (a) Set custom field wifidebug -> v15 on control queue
-
-        - name: Build custom-field payload (wifidebug -> version)
-          delegate_to: localhost
-          set_fact:
-            wdbg_cf_payload:
-              inscope_device: "{{ inscope_device_name }}"
-              task_name: "deploy_wifidebug"
-              task_result: "{{ wdbg_version }}"
-
-        - name: Publish custom-field update to control queue
-          delegate_to: localhost
-          uri:
-            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-            method: POST
-            user: "{{ rmq_user }}"
-            password: "{{ rmq_pass }}"
-            force_basic_auth: true
-            status_code: 200
-            headers:
-              content-type: "application/json"
-            body_format: json
-            body:
-              properties:
-                content_type: "application/json"
-              routing_key: "{{ control_queue }}"
-              payload: "{{ wdbg_cf_payload | to_json }}"
-              payload_encoding: "string"
-          register: rmq_cf
-          changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
-
-        # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
-
-        - name: Build actions list
-          set_fact:
-            _actions_list: >-
-              {{
-                []
-                + ((upload_wdbg | default(false) | bool)                       | ternary(['uploaded wifidebug.sh'], []))
-                + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
-                + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
-                + ((do_edit | default(false) | bool)                           | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
-                + ((do_edit | default(false) | bool)                           | ternary(['restarted services'], []))
-              }}
-
-        - name: Build actions string
-          set_fact:
-            _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
-
-
-        - name: Build wrap-up journal payload
-          delegate_to: localhost
-          set_fact:
-            wrap_payload:
-              inscope_device: "{{ inscope_device_name }}"
-              task_name: "journal_add"
-              task_result: >-
-                wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
-                — actions: {{ _actions_str }}
-
-        - name: Publish wrap-up journal to control queue
-          delegate_to: localhost
-          uri:
-            url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
-            method: POST
-            user: "{{ rmq_user }}"
-            password: "{{ rmq_pass }}"
-            force_basic_auth: true
-            status_code: 200
-            headers:
-              content-type: "application/json"
-            body_format: json
-            body:
-              properties:
-                content_type: "application/json"
-              routing_key: "{{ control_queue }}"
-              payload: "{{ wrap_payload | to_json }}"
-              payload_encoding: "string"
-          register: rmq_wrap
-          changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
-
-        # Local summary (kept for operator visibility)
-        - name: Summary
-          debug:
-            msg:
-              - "result_status: {{ result_status }}"
-              - "we're good"
-

files/rabbit-client.sh

@@ -64,24 +64,41 @@ dispatch_task() {
   task="$(jq -er '.task_name // empty' <<<"$json")" || task=""
   task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options=""
 
+
   # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth)
-  if [[ "$task" == "afterupgrade_check" ]]; then
-    local attempt corr_id orig_at target_ver
-    attempt="$(jq -r '.attempt // empty' <<<"$json")"
-    corr_id="$(jq -r '.correlation_id // empty' <<<"$json")"
-    orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")"
-    target_ver="$(jq -r '.target_version // empty' <<<"$json")"
+  # Supports both historic 'afterupgrade_check' and current 'afterupgrade_indoor_check'
+  if [[ "$task" == "afterupgrade_indoor_check" || "$task" == "afterupgrade_check" ]]; then
+    # Extract everything the publisher may send
+    local attempt max_attempts cur_delay corr_id orig_at target_ver tv_full schema
+    attempt="$(jq -r '.attempt // ""' <<<"$json")"
+    max_attempts="$(jq -r '.max_attempts // ""' <<<"$json")"
+    cur_delay="$(jq -r '.current_delay_sec // ""' <<<"$json")"
+    corr_id="$(jq -r '.correlation_id // ""' <<<"$json")"
+    orig_at="$(jq -r '.original_emitted_at // ""' <<<"$json")"
+    target_ver="$(jq -r '.target_version // ""' <<<"$json")"
+    tv_full="$(jq -r '.target_version_full // ""' <<<"$json")"
+    schema="$(jq -r '.schema_version // ""' <<<"$json")"
 
-    [[ -n "$attempt"  ]] && task_options+=" -e attempt=${attempt}"
-    [[ -n "$corr_id"  ]] && task_options+=" -e correlation_id=${corr_id}"
-    [[ -n "$orig_at"  ]] && task_options+=" -e original_emitted_at=${orig_at}"
+    # Backfill target_version from target_version_full if missing (e.g., fox200-2.2.1-r6801.bin → 2.2.1-r6801)
+    if [[ -z "$target_ver" && -n "$tv_full" ]]; then
+      target_ver="$(sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+-r[0-9]+).*/\1/p' <<<"$tv_full" || true)"
+    fi
 
-    if [[ -n "$target_ver" ]]; then
-      local esc_tv=${target_ver//\'/\'\"\'\"\'}   # replace ' with '\'' safely
-      task_options+=" -e target_version='${esc_tv}'"
-    fi
+    # Safe single-quote escaper for -e 'value'
+    esc() { local s="$1"; printf "%s" "${s//\'/\047}"; }
+
+    # Always pass the vars (even if empty), so the play never sees undefined
+    task_options+=" -e attempt='$(esc "$attempt")'"
+    task_options+=" -e max_attempts='$(esc "$max_attempts")'"
+    task_options+=" -e current_delay_sec='$(esc "$cur_delay")'"
+    task_options+=" -e correlation_id='$(esc "$corr_id")'"
+    task_options+=" -e original_emitted_at='$(esc "$orig_at")'"
+    task_options+=" -e target_version='$(esc "$target_ver")'"
+    task_options+=" -e target_version_full='$(esc "$tv_full")'"
+    task_options+=" -e schema_version='$(esc "$schema")'"
   fi
 
+
   if [[ -z "$device" || -z "$task" ]]; then
     warn "payload missing required keys (inscope_device/task_name). Skipping."
     return 0