From 9aadf7bb107f1c478b281fd13202f84a43ad8d89 Mon Sep 17 00:00:00 2001 From: pavel Date: Thu, 23 Oct 2025 05:09:48 +0300 Subject: [PATCH] 0509 --- .../afterupgrade_check.yml-from_chat | 355 ------ .../afterupgrade_check.yml-from_container | 432 ------- .../afterupgrade_check.yml-screewed | 359 ------ .../afterupgrade_indoor_check.yml | 7 +- .../afterupgrade_indoor_check.yml-bfr_3tries | 201 --- .../afterupgrade_indoor_check.yml-chat | 0 ...afterupgrade_indoor_check.yml-fixingerrors | 644 ---------- .../ansible-playbooks/update-indoor.yml-bckp1 | 997 --------------- .../update-indoor.yml-beforefixrebootin | 1108 ---------------- .../update-indoor.yml-bfr_reboot_fix | 1100 ---------------- .../ansible-playbooks/update-indoor.yml-day2 | 1117 ----------------- .../update-indoor.yml-fixingerrors | 1117 ----------------- files/ansible-playbooks/update-indoor.yml2 | 1108 ---------------- files/ansible-playbooks/wifidebug17.yml-old | 441 ------- 14 files changed, 4 insertions(+), 8982 deletions(-) delete mode 100644 files/ansible-playbooks/afterupgrade_check.yml-from_chat delete mode 100644 files/ansible-playbooks/afterupgrade_check.yml-from_container delete mode 100644 files/ansible-playbooks/afterupgrade_check.yml-screewed delete mode 100644 files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries delete mode 100644 files/ansible-playbooks/afterupgrade_indoor_check.yml-chat delete mode 100644 files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors delete mode 100644 files/ansible-playbooks/update-indoor.yml-bckp1 delete mode 100644 files/ansible-playbooks/update-indoor.yml-beforefixrebootin delete mode 100644 files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix delete mode 100644 files/ansible-playbooks/update-indoor.yml-day2 delete mode 100644 files/ansible-playbooks/update-indoor.yml-fixingerrors delete mode 100644 files/ansible-playbooks/update-indoor.yml2 delete mode 100644 files/ansible-playbooks/wifidebug17.yml-old diff --git a/files/ansible-playbooks/afterupgrade_check.yml-from_chat b/files/ansible-playbooks/afterupgrade_check.yml-from_chat deleted file mode 100644 index d7edd1b..0000000 --- a/files/ansible-playbooks/afterupgrade_check.yml-from_chat +++ /dev/null @@ -1,355 +0,0 @@ ---- -- name: After-upgrade verification (banner check + reporting) - hosts: all - gather_facts: no - - # RabbitMQ + defaults (match the big script) - vars: - rmq_host: "10.210.12.2" - rmq_port: 15672 - rmq_user: "admin" - rmq_pass: "change_me" - rmq_vhost: "app" - rmq_exchange: "controls" - control_queue: "queue_controls" - - # Probing/SSH defaults - tcp_port: 22 - nc_timeout: 5 - ssh_user: "{{ ansible_user | default('root') }}" - ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" - ssh_timeout: 10 - - # Do NOT self-reference max_attempts. We’ll normalize below. - max_attempts_default: 3 - - tasks: - # ---- Normalize metadata safely (no self-referential defaults) ---- - - name: Normalize metadata (no clever transforms) - ansible.builtin.set_fact: - attempt: "{{ (attempt | default(1)) | int }}" - effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" - correlation_id: "{{ correlation_id | default('') }}" - original_emitted_at: "{{ original_emitted_at | default('') }}" - target_version: "{{ target_version | default('') }}" - # preserve the original string verbatim for all subsequent retries - target_version_full: "{{ target_version | default('') }}" - - - name: Show received metadata - ansible.builtin.debug: - msg: - - "attempt={{ attempt }}" - - "max_attempts={{ effective_max_attempts }}" - - "correlation_id={{ correlation_id }}" - - "original_emitted_at={{ original_emitted_at }}" - - "target_version(full)={{ target_version_full }}" - - # ---- Fast TCP reachability probe (controller-side) ---- - - name: Check if TCP/{{ tcp_port }} is reachable with nc - delegate_to: localhost - ansible.builtin.shell: | - nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Build failure journal (no TCP connectivity) + mark retry - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed). - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (no TCP connectivity) - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_tcp_fail - changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) - - # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). - - name: Stop host after TCP failure (we’ll schedule or close out below) - when: nc_probe.rc != 0 - ansible.builtin.meta: noop - - # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- - - name: Probe banner via SSH from controller (classic extraction) - when: nc_probe.rc == 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ ssh_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ ssh_pass }}' \ - ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ - "${USER}@${HOST}" \ - "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" - register: banner_probe - changed_when: false - ignore_errors: true - - - name: Show the current version (banner line) - when: nc_probe.rc == 0 and banner_probe.rc == 0 - ansible.builtin.debug: - msg: "{{ banner_probe.stdout | trim }}" - - - name: Evaluate version match (full-string contains check) - when: nc_probe.rc == 0 and banner_probe.rc == 0 - ansible.builtin.set_fact: - version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" - - # ---- Journaling paths ---- - # Success: banner matches expected full target_version - - name: Build success journal payload - when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_success_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): - Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}' - Correlation={{ correlation_id }} Original={{ original_emitted_at }} - - - name: Publish success journal to control queue - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_success_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_success - changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) - - # NEW: send a control tag to clean up device state on success - - name: Build cleanup control payload (update_cleanup_success) - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.set_fact: - control_cleanup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "update_cleanup_success" - - - name: Publish cleanup control message to control queue - when: control_cleanup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ control_cleanup_payload | to_json }}" - payload_encoding: "string" - register: rmq_cleanup_success - changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) - - # Mismatch: reachable & banner read, but not equal to target_version - - name: Build mismatch journal payload - when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_mismatch_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): - Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' - Correlation={{ correlation_id }} Original={{ original_emitted_at }} - - - name: Publish mismatch journal to control queue - when: journal_mismatch_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_mismatch_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_mismatch - changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) - - # SSH error path: TCP OK, but SSH failed - - name: Build failure journal payload (ssh error) + mark retry - when: nc_probe.rc == 0 and banner_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - journal_fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): - {{ (banner_probe.stderr | default('') | trim) }} - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (ssh error) to control queue - when: nc_probe.rc == 0 and banner_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_fail - changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) - - # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- - - name: Compute next-attempt delay (ms) according to policy - when: (_needs_retry | default(false)) | bool - ansible.builtin.set_fact: - next_attempt: "{{ attempt | int + 1 }}" - next_delay_sec: >- - {% if attempt | int == 1 %} - 300 - {% elif attempt | int == 2 %} - 600 - {% else %} - 0 - {% endif %} - next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" - - # If we've reached the cap, send a final “gave up” journal and stop. - - name: Build final gave-up journal (max attempts reached) - when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - journal_gaveup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): - Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - - - name: Publish final gave-up journal - when: journal_gaveup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_gaveup_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_gaveup - changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) - - - name: Stop host after final gave-up - when: journal_gaveup_payload is defined - ansible.builtin.meta: end_host - - # Otherwise schedule the next attempt (only if we still have budget) - - name: Build delayed after-upgrade payload for next attempt - when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - delayed_payload: - task_name: "afterupgrade_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - attempt: "{{ next_attempt | int }}" - max_attempts: "{{ effective_max_attempts | int }}" - correlation_id: "{{ correlation_id }}" - original_emitted_at: "{{ original_emitted_at }}" - target_version: "{{ target_version_full }}" - current_delay_sec: "{{ next_delay_sec | int }}" - schema_version: 1 - - - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) - when: delayed_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - expiration: "{{ (next_delay_ms | int) | string }}" - correlation_id: "{{ correlation_id }}" - routing_key: "deviceconfig" - payload: "{{ delayed_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_next - changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) - - - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) - when: (_needs_retry | default(false)) | bool - ansible.builtin.meta: end_host diff --git a/files/ansible-playbooks/afterupgrade_check.yml-from_container b/files/ansible-playbooks/afterupgrade_check.yml-from_container deleted file mode 100644 index 03c5f29..0000000 --- a/files/ansible-playbooks/afterupgrade_check.yml-from_container +++ /dev/null @@ -1,432 +0,0 @@ ---- -- name: After-upgrade verification (banner check + reporting) - hosts: all - gather_facts: no - - # RabbitMQ + defaults (match the big script) - vars: - rmq_host: "10.210.12.2" - rmq_port: 15672 - rmq_user: "admin" - rmq_pass: "change_me" - rmq_vhost: "app" - rmq_exchange: "controls" - control_queue: "queue_controls" - - # Probing/SSH defaults - tcp_port: 22 - nc_timeout: 5 - ssh_user: "{{ ansible_user | default('root') }}" - ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" - ssh_timeout: 10 - - # Do NOT self-reference max_attempts. We’ll normalize below. - max_attempts_default: 3 - - tasks: - # ---- Normalize metadata safely (no self-referential defaults) ---- - - name: Normalize metadata (no clever transforms) - ansible.builtin.set_fact: - attempt: "{{ (attempt | default(1)) | int }}" - effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" - correlation_id: "{{ correlation_id | default('') }}" - original_emitted_at: "{{ original_emitted_at | default('') }}" - target_version: "{{ target_version | default('') }}" - # preserve the original string verbatim for all subsequent retries - target_version_full: "{{ target_version | default('') }}" - - - name: Show received metadata - ansible.builtin.debug: - msg: - - "attempt={{ attempt }}" - - "max_attempts={{ effective_max_attempts }}" - - "correlation_id={{ correlation_id }}" - - "original_emitted_at={{ original_emitted_at }}" - - "target_version(full)={{ target_version_full }}" - - # ---- Fast TCP reachability probe (controller-side) ---- - - name: Check if TCP/{{ tcp_port }} is reachable with nc - delegate_to: localhost - ansible.builtin.shell: | - nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Build failure journal (no TCP connectivity) + mark retry - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): - TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }} - Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (no TCP connectivity) - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_tcp_fail - changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) - - # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). - - name: Stop host after TCP failure (we’ll schedule or close out below) - when: nc_probe.rc != 0 - ansible.builtin.meta: noop - - # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- - - name: Probe banner via SSH from controller (classic extraction) - when: nc_probe.rc == 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ ssh_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ ssh_pass }}' \ - ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ - "${USER}@${HOST}" \ - "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" - register: banner_probe - changed_when: false - ignore_errors: true - - - name: Show the current version (banner line) - when: nc_probe.rc == 0 and banner_probe.rc == 0 - ansible.builtin.debug: - msg: "{{ banner_probe.stdout | trim }}" - - - name: Evaluate version match (full-string contains check) - when: nc_probe.rc == 0 and banner_probe.rc == 0 - ansible.builtin.set_fact: - version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" - - # ---- Journaling paths ---- - - # Success: banner matches expected full target_version - - name: Build success journal payload - when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_success_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): - Banner='{{ (banner_probe.stdout | default('') | trim) }}' - Target='{{ target_version_full }}' - Correlation={{ correlation_id }} Original={{ original_emitted_at }} - - - name: Publish success journal to control queue - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_success_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_success - changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) - - # NEW: send a control tag to clean up device state on success - - name: Build cleanup control payload (update_cleanup_success) - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.set_fact: - control_cleanup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "update_cleanup_success" - - - name: Publish cleanup control message to control queue - when: control_cleanup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ control_cleanup_payload | to_json }}" - payload_encoding: "string" - register: rmq_cleanup_success - changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) - - # --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag --- - - - name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801") - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.set_fact: - indoor_fwver_norm: >- - {{ - ( (dev2_fwver.stdout | default('') | trim) - | regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1') - | regex_replace('\\s+', ' ') - | trim - ) - }} - - - name: Publish NetBox custom field indoor_fwver to control queue - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: { content-type: "application/json" } - body_format: json - body: - properties: { content_type: "application/json" } - routing_key: "{{ control_queue }}" - payload: "{{ { - 'inscope_device': (ansible_hostname | default(inventory_hostname)), - 'task_name': 'custom_field_set', - 'task_add1': 'indoor_fwver', - 'task_result': (indoor_fwver_norm | default('')) - } | to_json }}" - payload_encoding: "string" - register: rmq_cf_indoor_fwver - changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool) - - - name: Build payload to remove indoor-restart-scheduled tag (DEV1) - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.set_fact: - tag_remove_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_remove" - task_result: "indoor-restart-scheduled" - - - name: Publish indoor-restart-scheduled tag removal - when: tag_remove_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_remove_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_remove_sched - changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) - - - # Mismatch: reachable & banner read, but not equal to target_version - - name: Build mismatch journal payload - when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_mismatch_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): - Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' - Correlation={{ correlation_id }} Original={{ original_emitted_at }} - - - name: Publish mismatch journal to control queue - when: journal_mismatch_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_mismatch_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_mismatch - changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) - - # SSH error path: TCP OK, but SSH failed - - name: Build failure journal payload (ssh error) + mark retry - when: nc_probe.rc == 0 and banner_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - journal_fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): - {{ (banner_probe.stderr | default('') | trim) }} - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (ssh error) to control queue - when: nc_probe.rc == 0 and banner_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_fail - changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) - - # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- - - - name: Compute next-attempt delay (ms) according to policy - when: (_needs_retry | default(false)) | bool - ansible.builtin.set_fact: - next_attempt: "{{ attempt | int + 1 }}" - next_delay_sec: >- - {% if attempt | int == 1 %} - 300 - {% elif attempt | int == 2 %} - 600 - {% else %} - 0 - {% endif %} - next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" - - # If we've reached the cap, send a final “gave up” journal and stop. - - name: Build final gave-up journal (max attempts reached) - when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - journal_gaveup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): - Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - - - name: Publish final gave-up journal - when: journal_gaveup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_gaveup_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_gaveup - changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) - - - name: Stop host after final gave-up - when: journal_gaveup_payload is defined - ansible.builtin.meta: end_host - - # Otherwise schedule the next attempt (only if we still have budget) - - name: Build delayed after-upgrade payload for next attempt - when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - delayed_payload: - task_name: "afterupgrade_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - attempt: "{{ next_attempt | int }}" - max_attempts: "{{ effective_max_attempts | int }}" - correlation_id: "{{ correlation_id }}" - original_emitted_at: "{{ original_emitted_at }}" - target_version: "{{ target_version_full }}" - current_delay_sec: "{{ next_delay_sec | int }}" - schema_version: 1 - - - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) - when: delayed_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - expiration: "{{ (next_delay_ms | int) | string }}" - correlation_id: "{{ correlation_id }}" - routing_key: "deviceconfig" - payload: "{{ delayed_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_next - changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) - - - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) - when: (_needs_retry | default(false)) | bool - ansible.builtin.meta: end_host - diff --git a/files/ansible-playbooks/afterupgrade_check.yml-screewed b/files/ansible-playbooks/afterupgrade_check.yml-screewed deleted file mode 100644 index e5cd2ce..0000000 --- a/files/ansible-playbooks/afterupgrade_check.yml-screewed +++ /dev/null @@ -1,359 +0,0 @@ ---- -- name: After-upgrade verification (banner check + reporting) - hosts: all - gather_facts: no - - # RabbitMQ + defaults (match the big script) - vars: - rmq_host: "10.210.12.2" - rmq_port: 15672 - rmq_user: "admin" - rmq_pass: "change_me" - rmq_vhost: "app" - rmq_exchange: "controls" - control_queue: "queue_controls" - - # Probing/SSH defaults - tcp_port: 22 - nc_timeout: 5 - ssh_user: "{{ ansible_user | default('root') }}" - ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" - ssh_timeout: 10 - - # Do NOT self-reference max_attempts. We’ll normalize below. - max_attempts_default: 3 - - tasks: - # ---- Normalize metadata safely (no self-referential defaults) ---- - - name: Normalize metadata (no clever transforms) - ansible.builtin.set_fact: - attempt: "{{ (attempt | default(1)) | int }}" - effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" - correlation_id: "{{ correlation_id | default('') }}" - original_emitted_at: "{{ original_emitted_at | default('') }}" - target_version: "{{ target_version | default('') }}" - # preserve the original string verbatim for all subsequent retries - target_version_full: "{{ target_version | default('') }}" - - - name: Show received metadata - ansible.builtin.debug: - msg: - - "attempt={{ attempt }}" - - "max_attempts={{ effective_max_attempts }}" - - "correlation_id={{ correlation_id }}" - - "original_emitted_at={{ original_emitted_at }}" - - "target_version(full)={{ target_version_full }}" - - # ---- Fast TCP reachability probe (controller-side) ---- - - name: Check if TCP/{{ tcp_port }} is reachable with nc - delegate_to: localhost - ansible.builtin.shell: | - nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Build failure journal (no TCP connectivity) + mark retry - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): - TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }} - Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (no TCP connectivity) - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_tcp_fail - changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) - - # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). - - name: Stop host after TCP failure (we’ll schedule or close out below) - when: nc_probe.rc != 0 - ansible.builtin.meta: noop - - # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- - - name: Probe banner via SSH from controller (classic extraction) - when: nc_probe.rc == 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ ssh_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ ssh_pass }}' \ - ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ - "${USER}@${HOST}" \ - "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" - register: banner_probe - changed_when: false - ignore_errors: true - - - name: Show the current version (banner line) - when: nc_probe.rc == 0 and banner_probe.rc == 0 - ansible.builtin.debug: - msg: "{{ banner_probe.stdout | trim }}" - - - name: Evaluate version match (full-string contains check) - when: nc_probe.rc == 0 and banner_probe.rc == 0 - ansible.builtin.set_fact: - version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" - - # ---- Journaling paths ---- - - # Success: banner matches expected full target_version - - name: Build success journal payload - when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_success_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): - Banner='{{ (banner_probe.stdout | default('') | trim) }}' - Target='{{ target_version_full }}' - Correlation={{ correlation_id }} Original={{ original_emitted_at }} - - - name: Publish success journal to control queue - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_success_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_success - changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) - - # NEW: send a control tag to clean up device state on success - - name: Build cleanup control payload (update_cleanup_success) - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.set_fact: - control_cleanup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "update_cleanup_success" - - - name: Publish cleanup control message to control queue - when: control_cleanup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ control_cleanup_payload | to_json }}" - payload_encoding: "string" - register: rmq_cleanup_success - changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) - - # Mismatch: reachable & banner read, but not equal to target_version - - name: Build mismatch journal payload - when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_mismatch_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): - Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' - Correlation={{ correlation_id }} Original={{ original_emitted_at }} - - - name: Publish mismatch journal to control queue - when: journal_mismatch_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_mismatch_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_mismatch - changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) - - # SSH error path: TCP OK, but SSH failed - - name: Build failure journal payload (ssh error) + mark retry - when: nc_probe.rc == 0 and banner_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - journal_fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): - {{ (banner_probe.stderr | default('') | trim) }} - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (ssh error) to control queue - when: nc_probe.rc == 0 and banner_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_fail - changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) - - # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- - - - name: Compute next-attempt delay (ms) according to policy - when: (_needs_retry | default(false)) | bool - ansible.builtin.set_fact: - next_attempt: "{{ attempt | int + 1 }}" - next_delay_sec: >- - {% if attempt | int == 1 %} - 300 - {% elif attempt | int == 2 %} - 600 - {% else %} - 0 - {% endif %} - next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" - - # If we've reached the cap, send a final “gave up” journal and stop. - - name: Build final gave-up journal (max attempts reached) - when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - journal_gaveup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): - Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - - - name: Publish final gave-up journal - when: journal_gaveup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_gaveup_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_gaveup - changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) - - - name: Stop host after final gave-up - when: journal_gaveup_payload is defined - ansible.builtin.meta: end_host - - # Otherwise schedule the next attempt (only if we still have budget) - - name: Build delayed after-upgrade payload for next attempt - when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - delayed_payload: - task_name: "afterupgrade_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - attempt: "{{ next_attempt | int }}" - max_attempts: "{{ effective_max_attempts | int }}" - correlation_id: "{{ correlation_id }}" - original_emitted_at: "{{ original_emitted_at }}" - target_version: "{{ target_version_full }}" - current_delay_sec: "{{ next_delay_sec | int }}" - schema_version: 1 - - - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) - when: delayed_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - expiration: "{{ (next_delay_ms | int) | string }}" - correlation_id: "{{ correlation_id }}" - routing_key: "deviceconfig" - payload: "{{ delayed_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_next - changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) - - - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) - when: (_needs_retry | default(false)) | bool - ansible.builtin.meta: end_host diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml b/files/ansible-playbooks/afterupgrade_indoor_check.yml index 3222d5d..b87bb0f 100644 --- a/files/ansible-playbooks/afterupgrade_indoor_check.yml +++ b/files/ansible-playbooks/afterupgrade_indoor_check.yml @@ -30,6 +30,8 @@ rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + debug_aic: false + tasks: # ---- Normalize metadata from delayed message ---- - name: Normalize after upgrade metadata into facts @@ -353,13 +355,12 @@ - name: Debug | banner normalization inputs delegate_to: localhost - when: debug_aic | bool + when: (debug_aic | default(false)) | bool ansible.builtin.debug: msg: - - "banner_raw={{ (firmware_banner.stdout | default(firmware_banner) | default('')) | trim }}" + - "banner_raw={{ (dev2_fwver.stdout | default('') | trim) }}" - "normalize rule: '2.2.1 rev 6801' -> '2.2.1-r6801'" - - name: Debug show raw firmware readout from DEV2 delegate_to: localhost ansible.builtin.debug: diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries b/files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries deleted file mode 100644 index 3051e9e..0000000 --- a/files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries +++ /dev/null @@ -1,201 +0,0 @@ ---- -- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel) - hosts: all - gather_facts: no - - vars: - # ---------------- BusyBox-safe path prefix ---------------- - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # ---------------- DEV1 (outer device) ---------------- - dev1_user: "root" - dev1_pass: "wavewave" - dev1_iface: "br-wan" - - # ---------------- DEV2 (indoor behind DEV1) ---------------- - dev2_host: "192.168.1.1" - dev2_port: 22 - dev2_side_ip: "192.168.1.11/24" - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # ---------------- RabbitMQ (same env scheme as main playbooks) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - - tasks: - # ---------------- Step 1: Read DEV1 hostname and sanity ---------------- - - name: Read DEV1 hostname - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 hostname mismatch - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - # ---------------- Step 2: Setup temporary IP for reachability ---------------- - - name: Add temporary IP on DEV1 (ignore if exists) - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 and - ('File exists' not in (add_ip.stdout | default(''))) and - ('File exists' not in (add_ip.stderr | default(''))) - - # ---------------- Step 3: Start SSH tunnel via DEV1 ---------------- - - name: Pick random free local port - delegate_to: localhost - ansible.builtin.shell: | - set -e - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n1)" - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi - done - exit 1 - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control dir - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - - name: Start SSH ControlMaster tunnel via DEV1 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sshpass -p '{{ dev1_pass }}' ssh -f -N \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \ - "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - # ---------------- Step 4: Determine working password for DEV2 ---------------- - - name: Try both passfiles for DEV2 - delegate_to: localhost - ansible.builtin.shell: | - for f in {{ dev2_passfiles | join(' ') }}; do - if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then - echo "$f"; exit 0; - fi - done - echo NONE - register: dev2_passfile_try - changed_when: false - - - name: Save selected DEV2 passfile - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}" - changed_when: false - - - name: Stop if no valid passfile found - ansible.builtin.meta: end_host - when: dev2_passfile_used == "NONE" - - # ---------------- Step 5: Read firmware version on DEV2 ---------------- - - name: Read firmware version from DEV2 - delegate_to: localhost - ansible.builtin.shell: | - sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \ - root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown" - register: dev2_fwver - changed_when: false - ignore_errors: true - - - name: Show firmware version readout - delegate_to: localhost - ansible.builtin.debug: - msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" - - # ---------------- Step 6: Publish result journal ---------------- - - name: Build journal payload - delegate_to: localhost - ansible.builtin.set_fact: - journal_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}' - tunnel_port={{ _local_port }} - - - name: Publish journal to control queue - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_fwver - changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool) - - post_tasks: - - name: Cleanup tunnel and temp IP - block: - - ansible.builtin.debug: - msg: "Cleaning up tunnel + temp IP" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP from DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - failed_when: false - changed_when: false diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml-chat b/files/ansible-playbooks/afterupgrade_indoor_check.yml-chat deleted file mode 100644 index e69de29..0000000 diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors b/files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors deleted file mode 100644 index 6797484..0000000 --- a/files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors +++ /dev/null @@ -1,644 +0,0 @@ ---- -- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel) - hosts: all - gather_facts: no - - vars: - # ---------------- BusyBox-safe path prefix ---------------- - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # ---------------- DEV1 (outer device) ---------------- - dev1_user: "root" - dev1_pass: "wavewave" - dev1_iface: "br-wan" - - # ---------------- DEV2 (indoor behind DEV1) ---------------- - dev2_host: "192.168.1.1" - dev2_port: 22 - dev2_side_ip: "192.168.1.11/24" - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # ---------------- RabbitMQ (same env scheme as main playbooks) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - - tasks: - # ---- Normalize metadata (from delayed message) ---- - - name: Normalize after-upgrade metadata - delegate_to: localhost - ansible.builtin.set_fact: - attempt: "{{ (attempt | default(1)) | int }}" - effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" - correlation_id: "{{ correlation_id | default('') }}" - original_emitted_at: "{{ original_emitted_at | default('') }}" - target_version: "{{ target_version | default('') }}" - target_version_full: "{{ target_version | default('') }}" - - # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ---- - - name: Check if TCP/22 on DEV1 is reachable - delegate_to: localhost - ansible.builtin.shell: | - nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22 - register: nc_probe - changed_when: false - ignore_errors: true - - # ---- If TCP down: journal + schedule next try or give up ---- - - name: Build failure journal (TCP unreachable) + mark retry - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.set_fact: - fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}): - TCP 22 unreachable. Correlation={{ correlation_id }} - Original={{ original_emitted_at }} Target='{{ target_version_full }}' - _needs_retry: true - - - name: Publish failure journal (TCP unreachable) - when: nc_probe.rc != 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: { content-type: "application/json" } - body_format: json - body: - properties: { content_type: "application/json" } - routing_key: "{{ control_queue }}" - payload: "{{ fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_tcp_fail - changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) - - - name: Compute next-attempt delay (10 minutes) and counters - when: (_needs_retry | default(false)) | bool - delegate_to: localhost - ansible.builtin.set_fact: - next_attempt: "{{ attempt | int + 1 }}" - next_delay_sec: 600 - next_delay_ms: 600000 - - - name: Build final gave-up journal (max attempts reached) - when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - journal_gaveup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): - Exhausted attempts. Last error path=TCP. - Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' - - - name: Publish final gave-up journal - when: journal_gaveup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: { content-type: "application/json" } - body_format: json - body: - properties: { content_type: "application/json" } - routing_key: "{{ control_queue }}" - payload: "{{ journal_gaveup_payload | to_json }}" - payload_encoding: "string" - register: rmq_j_gaveup - changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) - - - name: Stop host after final gave-up - when: journal_gaveup_payload is defined - ansible.builtin.meta: end_host - - - name: Build delayed after-upgrade payload for next attempt (10m) - when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) - delegate_to: localhost - ansible.builtin.set_fact: - delayed_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - attempt: "{{ next_attempt | int }}" - max_attempts: "{{ effective_max_attempts | int }}" - correlation_id: "{{ correlation_id }}" - original_emitted_at: "{{ original_emitted_at }}" - target_version: "{{ target_version_full }}" - current_delay_sec: "{{ next_delay_sec | int }}" - schema_version: 1 - - - name: Publish delayed next attempt to holding (dead-letters to deviceconfig) - when: delayed_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: { content-type: "application/json" } - body_format: json - body: - properties: - content_type: "application/json" - expiration: "{{ (next_delay_ms | int) | string }}" - correlation_id: "{{ correlation_id }}" - routing_key: "deviceconfig" - payload: "{{ delayed_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_next - changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) - - - name: Stop host after scheduling next attempt - when: (_needs_retry | default(false)) | bool - ansible.builtin.meta: end_host - - - # ---------------- Step 1: Read DEV1 hostname and sanity ---------------- - - name: Read DEV1 hostname - when: nc_probe.rc == 0 - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 hostname mismatch - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - # ---------------- Step 2: Setup temporary IP for reachability ---------------- - - name: Add temporary IP on DEV1 (ignore if exists) - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 and - ('File exists' not in (add_ip.stdout | default(''))) and - ('File exists' not in (add_ip.stderr | default(''))) - - # ---------------- Step 3: Start SSH tunnel via DEV1 ---------------- - - name: Pick random free local port - delegate_to: localhost - ansible.builtin.shell: | - set -e - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n1)" - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi - done - exit 1 - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control dir - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Start SSH ControlMaster tunnel via DEV1 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sshpass -p '{{ dev1_pass }}' ssh -f -N \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \ - "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) --- - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running (ssh -O check) - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - # ---------------- Step 4: Determine working password for DEV2 ---------------- - - name: Try both passfiles for DEV2 - delegate_to: localhost - ansible.builtin.shell: | - for f in {{ dev2_passfiles | join(' ') }}; do - if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then - echo "$f"; exit 0; - fi - done - echo NONE - register: dev2_passfile_try - changed_when: false - - - name: Save selected DEV2 passfile - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}" - changed_when: false - - - name: Debug selected DEV2 passfile - delegate_to: localhost - ansible.builtin.debug: - msg: "dev2_passfile_used={{ dev2_passfile_used }}" - - - name: Stop if no valid passfile found - ansible.builtin.meta: end_host - when: dev2_passfile_used == "NONE" - - # ---------------- Step 5: Read firmware version on DEV2 ---------------- - - name: Read firmware version from DEV2 - delegate_to: localhost - ansible.builtin.shell: | - sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \ - root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown" - register: dev2_fwver - changed_when: false - ignore_errors: true - - - name: Show firmware version readout - delegate_to: localhost - ansible.builtin.debug: - msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" - - # ---------------- Retry metadata + success evaluation ---------------- - - name: Normalize retry metadata for indoor checker - ansible.builtin.set_fact: - attempt: "{{ (attempt | default(1)) | int }}" - effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" - correlation_id: "{{ correlation_id | default('') }}" - original_emitted_at: "{{ original_emitted_at | default('') }}" - - - name: Evaluate indoor success (firmware read OK) - delegate_to: localhost - ansible.builtin.set_fact: - indoor_success: "{{ (dev2_fwver.rc | default(1) == 0) - and ((dev2_fwver.stdout | default('') | trim) | length > 0) - and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}" - - # ---------------- Step 6: Publish result journal (+ tags on success) ---------------- - # SUCCESS PATH: add tag, remove scheduled tag, and journal with "success" - - name: Build success journal payload - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.set_fact: - journal_success_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}' - tunnel_port={{ _local_port }} - Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - - name: Publish success journal to control queue - when: journal_success_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_success_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_success - changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool) - - - name: Build payload to add indoor-update-success tag (DEV1) - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.set_fact: - tag_add_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_add" - task_result: "indoor-update-success" - - - name: Publish indoor-update-success tag - when: tag_add_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_add_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_add_success - changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool) - - - name: Build payload to remove indoor-restart-scheduled tag (DEV1) - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.set_fact: - tag_remove_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_remove" - task_result: "indoor-restart-scheduled" - - - name: Publish indoor-restart-scheduled tag removal - when: tag_remove_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_remove_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_remove_sched - changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) - - # --- Normalize firmware string and set custom field on success --- - - name: Capture raw firmware banner from DEV2 (for normalization) - when: indoor_success | bool - delegate_to: localhost - ansible.builtin.set_fact: - fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}" - - - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801') - when: - - indoor_success | bool - - (fw_banner_raw | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.set_fact: - fw_norm: >- - {{ - fw_banner_raw - if (fw_banner_raw | lower is search('-r[0-9]+$')) - else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) - }} - - - name: Debug normalized firmware (indoor_fwver) - when: fw_norm is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')" - - - name: Publish custom_field_set indoor_fwver - when: - - indoor_success | bool - - fw_norm is defined - - (fw_norm | length) > 0 - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ { - 'inscope_device': (ansible_hostname | default(inventory_hostname)), - 'task_name': 'custom_field_set', - 'task_add1': 'indoor_fwver', - 'task_result': fw_norm - } | to_json }}" - payload_encoding: "string" - register: rmq_customfield_fw - changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool) - - # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart - - name: Build failure journal payload (indoor firmware read failed) - when: not (indoor_success | bool) - delegate_to: localhost - ansible.builtin.set_fact: - journal_fail_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}): - fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}' - Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - - name: Publish failure journal to control queue - when: journal_fail_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_fail_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_fail - changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool) - - - name: Compute retry parameters (10 minutes) - when: not (indoor_success | bool) - ansible.builtin.set_fact: - next_attempt: "{{ (attempt | int) + 1 }}" - next_delay_sec: 600 - next_delay_ms: "{{ 600000 }}" - - - name: Build final gave-up journal (max attempts reached) - when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int)) - delegate_to: localhost - ansible.builtin.set_fact: - journal_gaveup_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): - Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}. - Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} - - - name: Publish final gave-up journal - when: journal_gaveup_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_gaveup_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_gaveup - changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool) - - # Only schedule next attempt if we still have budget left - - name: Build delayed payload for next indoor attempt (10 min) - when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int)) - delegate_to: localhost - ansible.builtin.set_fact: - delayed_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - attempt: "{{ next_attempt | int }}" - max_attempts: "{{ effective_max_attempts | int }}" - correlation_id: "{{ correlation_id | default('') }}" - original_emitted_at: "{{ original_emitted_at | default('') }}" - current_delay_sec: "{{ next_delay_sec | int }}" - schema_version: 1 - - - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig) - when: delayed_payload is defined - delegate_to: localhost - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - expiration: "{{ (next_delay_ms | int) | string }}" - correlation_id: "{{ correlation_id | default('') }}" - routing_key: "deviceconfig" - payload: "{{ delayed_payload | to_json }}" - payload_encoding: "string" - register: rmq_pub_next - changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) - - post_tasks: - - name: Cleanup tunnel and temp IP - block: - - ansible.builtin.debug: - msg: "Cleaning up tunnel + temp IP" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP from DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - failed_when: false - changed_when: false - diff --git a/files/ansible-playbooks/update-indoor.yml-bckp1 b/files/ansible-playbooks/update-indoor.yml-bckp1 deleted file mode 100644 index 0574ccc..0000000 --- a/files/ansible-playbooks/update-indoor.yml-bckp1 +++ /dev/null @@ -1,997 +0,0 @@ ---- -# update-secondline.yml -- name: Second-line upgrade tunneled noninvasive engine - hosts: all - gather_facts: no - - vars: - # Busybox-safe PATH prefix for all remote raw calls on DEV1 - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # DEV1 credentials (stable, like rebootin222) - dev1_user: "root" - dev1_pass: "wavewave" - - # Tunnel target DEV2 behind DEV1 - dev2_host: "192.168.1.1" - dev2_port: 22 - - # Temp IP we add to DEV1 so it can reach DEV2 - dev2_side_ip: "192.168.1.11/24" - dev1_iface: "br-wan" - - # DEV2 behind the tunnel - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # SSH options used from controller - ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" - - # Image to stage on DEV2 noninvasive we do not run update -w here - image_filename: "fox200-2.2.1-r6801.bin" - image_md5: "56b7211709de617e058b98d4204e2562" - # Optional SHA256; leave empty to skip SHA256 checks - image_sha256: "" - dev2_image_dir: "/tmp" - dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" - - # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 - rebootin: "" - - # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - - pre_tasks: - # ------------------------------- Hostname sanity DEV1 ------------------------------- - - name: Read DEV1 hostname from env busybox safe - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 connected hostname differs from inventory - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - tasks: - # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - - name: Add temporary IP on DEV1 idempotent treat File exists as OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 - and ('File exists' not in (add_ip.stdout | default(''))) - and ('File exists' not in (add_ip.stderr | default(''))) - - - name: Debug add ip results - ansible.builtin.debug: - msg: - - "add_ip.rc={{ add_ip.rc | default('') }}" - - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" - - # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb - ansible.builtin.raw: > - {{ pathprefix }} - bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 - register: dev2_mac_scan - changed_when: false - - - name: Set dev2_mac from fdb scan (if any) - ansible.builtin.set_fact: - dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" - changed_when: false - - - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true - register: dev2_arp_del - changed_when: false - failed_when: false - - - name: Add static ARP for DEV2 on DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent - register: dev2_arp_add - changed_when: dev2_arp_add.rc == 0 - failed_when: > - dev2_arp_add.rc != 0 - and ('File exists' not in (dev2_arp_add.stdout | default(''))) - and ('File exists' not in (dev2_arp_add.stderr | default(''))) - - - name: Debug static ARP result - ansible.builtin.debug: - msg: - - "dev2_mac={{ dev2_mac | default('UNSET') }}" - - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - - - name: Skip ARP add because MAC not found - when: dev2_mac is not defined or dev2_mac | length == 0 - ansible.builtin.debug: - msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" - - # ---------------------------- Local tunnel preparation ---------------------------- - - name: Pick a free local port for the tunnel controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - pick() { - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n 1)" - if command -v ss >/dev/null 2>&1; then - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then - echo "$p"; return 0 - fi - else - if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then - echo "$p"; return 0 - fi - fi - done - return 1 - } - pick - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control directory - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Debug chosen tunnel params - delegate_to: localhost - ansible.builtin.debug: - msg: - - "local_port={{ _local_port }}" - - "ctrl_dir={{ _ctrl_dir }}" - - "ctrl_sock={{ _ctrl_sock }}" - - - name: issue arping with own IP - ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 - - # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - - name: Start SSH tunnel via DEV1 background ControlMaster - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ dev1_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - "${USER}@${HOST}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running proper O check with destination - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - - name: Sanity show presence and permissions of passfiles on controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" - register: dev2_ls - changed_when: false - - - name: Debug passfiles listing - delegate_to: localhost - ansible.builtin.debug: - msg: - - "{{ (dev2_ls.stdout | default('')) | trim }}" - - "{{ (dev2_ls.stderr | default('')) | trim }}" - - # ---------------------------- Pick DEV2 password root only ---------------------------- - - name: Try DEV2 login with basicpass root - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass - changed_when: false - ignore_errors: true - - - name: Select basicpass as working passfile - when: dev2_try_basicpass.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass" - changed_when: false - - - name: Try DEV2 login with basicpass2 root only if first failed - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass2 ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass2 - changed_when: false - ignore_errors: true - - - name: Select basicpass2 as working passfile - when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass2" - changed_when: false - - - name: Set dev2_passfile_used to NONE if neither worked - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "NONE" - changed_when: false - - - name: Debug selected DEV2 passfile or NONE - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - - - name: Read DEV2 hostname via tunnel (busybox-safe) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" - args: - executable: /bin/bash - register: dev2_host_read - changed_when: false - - - name: Normalize hostnames for strict compare - ansible.builtin.set_fact: - _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" - _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - - - name: Debug hostname bytes (hex) - delegate_to: localhost - ansible.builtin.debug: - msg: - - "inv={{ _inv_hn | tojson }}" - - "dev1={{ _dev1_hn | tojson }}" - - "dev2={{ _dev2_hn | tojson }}" - - # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - - name: Enforce DEV2 hostname equals inventory and DEV1 - block: - - name: Fail if DEV2 hostname differs from inventory/DEV1 - ansible.builtin.fail: - msg: > - Hostname mismatch: DEV2='{{ _dev2_hn }}', - inventory='{{ _inv_hn }}', - DEV1='{{ _dev1_hn }}' - when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) - rescue: - - name: Initialize journal array for hostname mismatch - ansible.builtin.set_fact: - _journal: [] - _blocked: true - _prep_blocked: false - delegate_to: localhost - - - name: Append hostname mismatch to journal - ansible.builtin.set_fact: - _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" - delegate_to: localhost - - - name: Build control queue payload for indoor aborted journal (hostname) - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue (hostname) - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_hn_resp - changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_hn_resp.status != 200) or - (rmq_journal_indoor_aborted_hn_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) - delegate_to: localhost - - - name: Stop host after hostname mismatch - ansible.builtin.meta: end_host - - # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - - name: Initialize journal and flags - ansible.builtin.set_fact: - _journal: [] - _prep_blocked: false - _blocked: false - - - name: Build specific marker filename (no .bin) - ansible.builtin.set_fact: - _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - - - name: Check for any preparation markers on DEV2 (wildcard count) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "ls /tmp/prepared_for* 2>/dev/null | wc -l" - args: - executable: /bin/bash - register: dev2_prep_count - changed_when: false - ignore_errors: true - - - name: Soft-block if any prep markers already present - when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - - # ---------------------------- DEV2 version firmux primary check ---------------------------- - - name: Read DEV2 firmux release - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /usr/lib/release/firmux 2>/dev/null || true" - args: - executable: /bin/bash - register: dev2_firmux - changed_when: false - ignore_errors: true - - - name: Debug firmux content DEV2 - when: dev2_firmux is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - # ---------------------------- Normalize rebootin early ---------------------------- - - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" - _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" - _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" - _reboot_minutes: >- - {{ - 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) - else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) - }} - - - name: "Normalize rebootin (phase 2: requested flag)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" - - # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - - name: Build control queue payload for indoor start journal - ansible.builtin.set_fact: - journal_indoor_start: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 is reachable, starting with update. - image={{ image_filename }}, md5={{ image_md5 }}, - firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, - reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }} - when: dev2_passfile_used != "NONE" - delegate_to: localhost - - - name: Publish indoor start journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_start | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_start_resp - changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_start_resp.status != 200) or - (rmq_journal_indoor_start_resp.json is not defined) or - (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) - when: journal_indoor_start is defined - delegate_to: localhost - - # ---------------------------- Stage image on DEV2 no write ---------------------------- - - name: Check local presence of image file - delegate_to: localhost - ansible.builtin.stat: - path: "{{ image_filename }}" - register: local_img - - - name: Soft-block if image missing locally - when: not local_img.stat.exists - ansible.builtin.set_fact: - _blocked: true - _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - - - name: Compute local md5 of the image - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.shell: | - set -e - md5sum "{{ image_filename }}" | awk '{print $1}' - register: local_md5 - changed_when: false - - - name: Verify local md5 matches expected - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.assert: - that: - - (local_md5.stdout | trim) == image_md5 - fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" - success_msg: "Local md5 matches expected" - - # Optional SHA256 local - - name: Compute local sha256 of the image (if provided) - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sha256sum "{{ image_filename }}" | awk '{print $1}' - register: local_sha256 - changed_when: false - ignore_errors: true - - - name: Soft-block if local sha256 mismatch or unavailable - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - # fw_printenv health before upload (soft-fail) - - name: Read fw_printenv size (line count) on DEV2 - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "(fw_printenv 2>/dev/null | wc -l) || echo 0" - args: - executable: /bin/bash - register: dev2_fwenv_wc - changed_when: false - ignore_errors: true - - - name: Soft-block if fw_printenv too small - when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - - - name: Check existing image md5 on DEV2 NOFILE if absent - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_before - changed_when: false - - - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" scp \ - -P "$PORT" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" - args: - executable: /bin/bash - register: scp_push - changed_when: true - - - name: Compute md5 of image on DEV2 after copy or if exists - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_after - changed_when: false - - - name: Verify DEV2 md5 matches expected (soft-fail journal) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" - - # Optional SHA256 remote - - name: Compute sha256 of image on DEV2 (if provided) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" - args: - executable: /bin/bash - register: dev2_sha256_after - changed_when: false - ignore_errors: true - - - name: Soft-block if DEV2 sha256 mismatch/unavailable - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - - name: Run noninvasive update check on DEV2 update c - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -c '{{ dev2_image_path }}' 2>&1 || true" - args: - executable: /bin/bash - register: dev2_update_check - changed_when: false - - - name: Debug update c output DEV2 - when: dev2_update_check is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - - - name: Soft-block if update -c did not say valid - when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" - - # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - - name: Build control queue payload for indoor aborted journal - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - when: (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_resp - changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_resp.status != 200) or - (rmq_journal_indoor_aborted_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) - when: journal_indoor_aborted is defined - delegate_to: localhost - - # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - - name: Upgrade write and flip block - when: not (_blocked | default(false)) - block: - - name: Write image on DEV2 (this will take a while) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') - - - name: Read current active partition on DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_before - changed_when: false - failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - - - name: Determine new active value for DEV2 - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) - ansible.builtin.set_fact: - dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - - - name: Switch active partition to {{ dev2_new_active }} - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_setenv active {{ dev2_new_active }}" - args: - executable: /bin/bash - register: dev2_setenv_out - changed_when: true - - - name: Verify active partition flipped on DEV2 - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_after - changed_when: false - failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - - - name: Create specific prep marker on DEV2 (no generic) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "touch '{{ _marker_specific }}'" - args: - executable: /bin/bash - register: dev2_marker_write - changed_when: true - ignore_errors: true - - # ---------------------------- Reboot scheduling (normalized) ---------------------------- - - name: Schedule delayed reboot on DEV2 minutes - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - SECS="$(( {{ _reboot_minutes | int }} * 60 ))" - CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" - args: - executable: /bin/bash - register: dev2_reboot_sched - changed_when: true - ignore_errors: true - - - name: Note reboot request bad format - when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested - ansible.builtin.debug: - msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" - - # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - - name: Build control queue payload for indoor updated journal - ansible.builtin.set_fact: - _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" - when: not (_blocked | default(false)) - delegate_to: localhost - - - name: Build indoor updated journal payload text - ansible.builtin.set_fact: - journal_indoor_updated: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 updated, scheduling reboot {{ - 'now' if (_reboot_requested and (_reboot_minutes | int) == 0) - else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) - else 'not requested' - }}. - write_done={{ _write_success }}, - active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, - active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Publish indoor updated journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_updated | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_updated_resp - changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_updated_resp.status != 200) or - (rmq_journal_indoor_updated_resp.json is not defined) or - (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) - when: journal_indoor_updated is defined - delegate_to: localhost - - # ---------------------------- summary ---------------------------- - - name: Summary show key results - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - "local_image_present={{ local_img.stat.exists | default(false) }}" - - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - - "reboot_requested={{ _reboot_requested | default(false) }}" - - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - - "prep_blocked={{ _prep_blocked | default(false) }}" - - "blocked={{ _blocked | default(false) }}" - - "journal={{ (_journal | default([])) | join(' || ') }}" - - post_tasks: - - name: Cleanup always - block: - - ansible.builtin.debug: - msg: "entering cleanup block" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel best effort - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir best effort - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - changed_when: del_ip.rc == 0 - failed_when: > - del_ip.rc != 0 - and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) - and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - - - name: Debug del ip results - ansible.builtin.debug: - msg: - - "del_ip.rc={{ del_ip.rc | default('') }}" - - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" - when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-beforefixrebootin b/files/ansible-playbooks/update-indoor.yml-beforefixrebootin deleted file mode 100644 index 96643cb..0000000 --- a/files/ansible-playbooks/update-indoor.yml-beforefixrebootin +++ /dev/null @@ -1,1108 +0,0 @@ ---- -# update-secondline.yml -- name: Second-line upgrade tunneled noninvasive engine - hosts: all - gather_facts: no - - vars: - # Busybox-safe PATH prefix for all remote raw calls on DEV1 - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # DEV1 credentials (stable, like rebootin222) - dev1_user: "root" - dev1_pass: "wavewave" - - # Tunnel target DEV2 behind DEV1 - dev2_host: "192.168.1.1" - dev2_port: 22 - - # Temp IP we add to DEV1 so it can reach DEV2 - dev2_side_ip: "192.168.1.11/24" - dev1_iface: "br-wan" - - # DEV2 behind the tunnel - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # SSH options used from controller - ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" - - # Image to stage on DEV2 noninvasive we do not run update -w here - image_filename: "fox200-2.2.1-r6801.bin" - image_md5: "56b7211709de617e058b98d4204e2562" - # Optional SHA256; leave empty to skip SHA256 checks - image_sha256: "" - dev2_image_dir: "/tmp" - dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" - - # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 - rebootin: "" - - # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" - - - pre_tasks: - # ------------------------------- Hostname sanity DEV1 ------------------------------- - - name: Read DEV1 hostname from env busybox safe - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 connected hostname differs from inventory - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - tasks: - # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - - name: Add temporary IP on DEV1 idempotent treat File exists as OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 - and ('File exists' not in (add_ip.stdout | default(''))) - and ('File exists' not in (add_ip.stderr | default(''))) - - - name: Debug add ip results - ansible.builtin.debug: - msg: - - "add_ip.rc={{ add_ip.rc | default('') }}" - - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" - - # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb - ansible.builtin.raw: > - {{ pathprefix }} - bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 - register: dev2_mac_scan - changed_when: false - - - name: Set dev2_mac from fdb scan (if any) - ansible.builtin.set_fact: - dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" - changed_when: false - - - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true - register: dev2_arp_del - changed_when: false - failed_when: false - - - name: Add static ARP for DEV2 on DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent - register: dev2_arp_add - changed_when: dev2_arp_add.rc == 0 - failed_when: > - dev2_arp_add.rc != 0 - and ('File exists' not in (dev2_arp_add.stdout | default(''))) - and ('File exists' not in (dev2_arp_add.stderr | default(''))) - - - name: Debug static ARP result - ansible.builtin.debug: - msg: - - "dev2_mac={{ dev2_mac | default('UNSET') }}" - - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - - - name: Skip ARP add because MAC not found - when: dev2_mac is not defined or dev2_mac | length == 0 - ansible.builtin.debug: - msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" - - # ---------------------------- Local tunnel preparation ---------------------------- - - name: Pick a free local port for the tunnel controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - pick() { - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n 1)" - if command -v ss >/dev/null 2>&1; then - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then - echo "$p"; return 0 - fi - else - if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then - echo "$p"; return 0 - fi - fi - done - return 1 - } - pick - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control directory - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Debug chosen tunnel params - delegate_to: localhost - ansible.builtin.debug: - msg: - - "local_port={{ _local_port }}" - - "ctrl_dir={{ _ctrl_dir }}" - - "ctrl_sock={{ _ctrl_sock }}" - - - name: issue arping with own IP - ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 - - # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - - name: Start SSH tunnel via DEV1 background ControlMaster - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ dev1_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - "${USER}@${HOST}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running proper O check with destination - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - - name: Sanity show presence and permissions of passfiles on controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" - register: dev2_ls - changed_when: false - - - name: Debug passfiles listing - delegate_to: localhost - ansible.builtin.debug: - msg: - - "{{ (dev2_ls.stdout | default('')) | trim }}" - - "{{ (dev2_ls.stderr | default('')) | trim }}" - - # ---------------------------- Pick DEV2 password root only ---------------------------- - - name: Try DEV2 login with basicpass root - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass - changed_when: false - ignore_errors: true - - - name: Select basicpass as working passfile - when: dev2_try_basicpass.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass" - changed_when: false - - - name: Try DEV2 login with basicpass2 root only if first failed - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass2 ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass2 - changed_when: false - ignore_errors: true - - - name: Select basicpass2 as working passfile - when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass2" - changed_when: false - - - name: Set dev2_passfile_used to NONE if neither worked - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "NONE" - changed_when: false - - - name: Debug selected DEV2 passfile or NONE - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - - - name: Read DEV2 hostname via tunnel (busybox-safe) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" - args: - executable: /bin/bash - register: dev2_host_read - changed_when: false - - - name: Normalize hostnames for strict compare - ansible.builtin.set_fact: - _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" - _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - - - name: Debug hostname bytes (hex) - delegate_to: localhost - ansible.builtin.debug: - msg: - - "inv={{ _inv_hn | tojson }}" - - "dev1={{ _dev1_hn | tojson }}" - - "dev2={{ _dev2_hn | tojson }}" - - # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - - name: Enforce DEV2 hostname equals inventory and DEV1 - block: - - name: Fail if DEV2 hostname differs from inventory/DEV1 - ansible.builtin.fail: - msg: > - Hostname mismatch: DEV2='{{ _dev2_hn }}', - inventory='{{ _inv_hn }}', - DEV1='{{ _dev1_hn }}' - when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) - rescue: - - name: Initialize journal array for hostname mismatch - ansible.builtin.set_fact: - _journal: [] - _blocked: true - _prep_blocked: false - delegate_to: localhost - - - name: Append hostname mismatch to journal - ansible.builtin.set_fact: - _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" - delegate_to: localhost - - - name: Build control queue payload for indoor aborted journal (hostname) - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue (hostname) - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_hn_resp - changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_hn_resp.status != 200) or - (rmq_journal_indoor_aborted_hn_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) - delegate_to: localhost - - - name: Stop host after hostname mismatch - ansible.builtin.meta: end_host - - # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - - name: Initialize journal and flags - ansible.builtin.set_fact: - _journal: [] - _prep_blocked: false - _blocked: false - - - name: Build specific marker filename (no .bin) - ansible.builtin.set_fact: - _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - - - name: Check for any preparation markers on DEV2 (wildcard count) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "ls /tmp/prepared_for* 2>/dev/null | wc -l" - args: - executable: /bin/bash - register: dev2_prep_count - changed_when: false - ignore_errors: true - - - name: Soft-block if any prep markers already present - when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - - # ---------------------------- DEV2 version firmux primary check ---------------------------- - - name: Read DEV2 firmux release - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /usr/lib/release/firmux 2>/dev/null || true" - args: - executable: /bin/bash - register: dev2_firmux - changed_when: false - ignore_errors: true - - - name: Debug firmux content DEV2 - when: dev2_firmux is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - # ---------------------------- Normalize rebootin early ---------------------------- - - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" - _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" - _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" - _reboot_minutes: >- - {{ - 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) - else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) - }} - - - name: "Normalize rebootin (phase 2: requested flag)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" - - # >>> NEW: compute effective seconds (0 min → 20 seconds) - - name: "Normalize rebootin (phase 3: compute seconds)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_seconds: >- - {{ - 20 if (_reboot_requested | default(false)) and ((_reboot_minutes | int) == 0) - else ((_reboot_minutes | int) * 60) - }} - - # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - - name: Build control queue payload for indoor start journal - ansible.builtin.set_fact: - journal_indoor_start: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 is reachable, starting with update. - image={{ image_filename }}, md5={{ image_md5 }}, - firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, - reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} - when: dev2_passfile_used != "NONE" - delegate_to: localhost - - - name: Publish indoor start journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_start | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_start_resp - changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_start_resp.status != 200) or - (rmq_journal_indoor_start_resp.json is not defined) or - (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) - when: journal_indoor_start is defined - delegate_to: localhost - - # ---------------------------- Stage image on DEV2 no write ---------------------------- - - name: Check local presence of image file - delegate_to: localhost - ansible.builtin.stat: - path: "{{ image_filename }}" - register: local_img - - - name: Soft-block if image missing locally - when: not local_img.stat.exists - ansible.builtin.set_fact: - _blocked: true - _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - - - name: Compute local md5 of the image - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.shell: | - set -e - md5sum "{{ image_filename }}" | awk '{print $1}' - register: local_md5 - changed_when: false - - - name: Verify local md5 matches expected - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.assert: - that: - - (local_md5.stdout | trim) == image_md5 - fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" - success_msg: "Local md5 matches expected" - - # Optional SHA256 local - - name: Compute local sha256 of the image (if provided) - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sha256sum "{{ image_filename }}" | awk '{print $1}' - register: local_sha256 - changed_when: false - ignore_errors: true - - - name: Soft-block if local sha256 mismatch or unavailable - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - # fw_printenv health before upload (soft-fail) - - name: Read fw_printenv size (line count) on DEV2 - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "(fw_printenv 2>/dev/null | wc -l) || echo 0" - args: - executable: /bin/bash - register: dev2_fwenv_wc - changed_when: false - ignore_errors: true - - - name: Soft-block if fw_printenv too small - when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - - - name: Check existing image md5 on DEV2 NOFILE if absent - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_before - changed_when: false - - - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" scp \ - -P "$PORT" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" - args: - executable: /bin/bash - register: scp_push - changed_when: true - - - name: Compute md5 of image on DEV2 after copy or if exists - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_after - changed_when: false - - - name: Verify DEV2 md5 matches expected (soft-fail journal) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" - - # Optional SHA256 remote - - name: Compute sha256 of image on DEV2 (if provided) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" - args: - executable: /bin/bash - register: dev2_sha256_after - changed_when: false - ignore_errors: true - - - name: Soft-block if DEV2 sha256 mismatch/unavailable - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - - name: Run noninvasive update check on DEV2 update c - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -c '{{ dev2_image_path }}' 2>&1 || true" - args: - executable: /bin/bash - register: dev2_update_check - changed_when: false - - - name: Debug update c output DEV2 - when: dev2_update_check is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - - - name: Soft-block if update -c did not say valid - when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" - - # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - - name: Build control queue payload for indoor aborted journal - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - when: (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_resp - changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_resp.status != 200) or - (rmq_journal_indoor_aborted_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) - when: journal_indoor_aborted is defined - delegate_to: localhost - - # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - - name: Upgrade write and flip block - when: not (_blocked | default(false)) - block: - - name: Write image on DEV2 (this will take a while) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') - - - name: Read current active partition on DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_before - changed_when: false - failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - - - name: Determine new active value for DEV2 - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) - ansible.builtin.set_fact: - dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - - - name: Switch active partition to {{ dev2_new_active }} - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_setenv active {{ dev2_new_active }}" - args: - executable: /bin/bash - register: dev2_setenv_out - changed_when: true - - - name: Verify active partition flipped on DEV2 - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_after - changed_when: false - failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - - - name: Create specific prep marker on DEV2 (no generic) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "touch '{{ _marker_specific }}'" - args: - executable: /bin/bash - register: dev2_marker_write - changed_when: true - ignore_errors: true - - # ---------------------------- Reboot scheduling (normalized) ---------------------------- - - name: Schedule delayed reboot on DEV2 minutes - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - SECS="{{ _reboot_seconds | int }}" - CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" - args: - executable: /bin/bash - register: dev2_reboot_sched - changed_when: true - ignore_errors: true - - - name: Build control queue payload for indoor-restart-scheduled tag - ansible.builtin.set_fact: - tag_restart_sched_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_add" - task_result: "indoor-restart-scheduled" - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_restart_sched_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_restart_sched_resp - changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_tag_restart_sched_resp.status != 200) or - (rmq_tag_restart_sched_resp.json is not defined) or - (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) - when: tag_restart_sched_payload is defined - delegate_to: localhost - - - - name: Note reboot request bad format - when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested - ansible.builtin.debug: - msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" - - # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - - name: Build control queue payload for indoor updated journal - ansible.builtin.set_fact: - _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" - when: not (_blocked | default(false)) - delegate_to: localhost - - - name: Build indoor updated journal payload text - ansible.builtin.set_fact: - journal_indoor_updated: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 updated, scheduling reboot {{ - ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) - else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) - else 'not requested' - }}. - write_done={{ _write_success }}, - active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, - active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Publish indoor updated journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_updated | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_updated_resp - changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_updated_resp.status != 200) or - (rmq_journal_indoor_updated_resp.json is not defined) or - (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) - when: journal_indoor_updated is defined - delegate_to: localhost - - - - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) - ansible.builtin.set_fact: - au_attempt: 1 - au_max_attempts: 3 - au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Generate correlation ID and original emitted timestamp - ansible.builtin.set_fact: - au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" - au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" - when: au_delay_sec is defined - delegate_to: localhost - - - name: Build after-upgrade check payload (attempt 1) - ansible.builtin.set_fact: - afterupgrade_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" - attempt: "{{ au_attempt }}" - max_attempts: "{{ au_max_attempts }}" - current_delay_sec: "{{ au_delay_sec }}" - correlation_id: "{{ au_correlation_id }}" - original_emitted_at: "{{ au_original_emitted_at }}" - schema_version: 1 - when: au_delay_sec is defined - delegate_to: localhost - - - name: Debug x-delay about to be sent (ms) - ansible.builtin.debug: - msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" - when: afterupgrade_payload is defined - delegate_to: localhost - - - name: Publish delayed after-upgrade check to holding exchange - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - headers: - x-delay: "{{ (au_delay_sec | int) * 1000 }}" - routing_key: "{{ afterupgrade_routing_key }}" - payload: "{{ afterupgrade_payload | to_json }}" - payload_encoding: "string" - register: rmq_afterupgrade_resp - changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_afterupgrade_resp.status != 200) or - (rmq_afterupgrade_resp.json is not defined) - when: afterupgrade_payload is defined - delegate_to: localhost - - - # ---------------------------- summary ---------------------------- - - name: Summary show key results - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - "local_image_present={{ local_img.stat.exists | default(false) }}" - - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - - "reboot_requested={{ _reboot_requested | default(false) }}" - - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - - "prep_blocked={{ _prep_blocked | default(false) }}" - - "blocked={{ _blocked | default(false) }}" - - "journal={{ (_journal | default([])) | join(' || ') }}" - - post_tasks: - - name: Cleanup always - block: - - ansible.builtin.debug: - msg: "entering cleanup block" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel best effort - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir best effort - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - changed_when: del_ip.rc == 0 - failed_when: > - del_ip.rc != 0 - and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) - and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - - - name: Debug del ip results - ansible.builtin.debug: - msg: - - "del_ip.rc={{ del_ip.rc | default('') }}" - - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" - when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix b/files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix deleted file mode 100644 index 1b93082..0000000 --- a/files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix +++ /dev/null @@ -1,1100 +0,0 @@ ---- -# update-secondline.yml -- name: Second-line upgrade tunneled noninvasive engine - hosts: all - gather_facts: no - - vars: - # Busybox-safe PATH prefix for all remote raw calls on DEV1 - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # DEV1 credentials (stable, like rebootin222) - dev1_user: "root" - dev1_pass: "wavewave" - - # Tunnel target DEV2 behind DEV1 - dev2_host: "192.168.1.1" - dev2_port: 22 - - # Temp IP we add to DEV1 so it can reach DEV2 - dev2_side_ip: "192.168.1.11/24" - dev1_iface: "br-wan" - - # DEV2 behind the tunnel - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # SSH options used from controller - ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" - - # Image to stage on DEV2 noninvasive we do not run update -w here - image_filename: "fox200-2.2.1-r6801.bin" - image_md5: "56b7211709de617e058b98d4204e2562" - # Optional SHA256; leave empty to skip SHA256 checks - image_sha256: "" - dev2_image_dir: "/tmp" - dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" - - # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 - rebootin: "" - - # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" - - - pre_tasks: - # ------------------------------- Hostname sanity DEV1 ------------------------------- - - name: Read DEV1 hostname from env busybox safe - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 connected hostname differs from inventory - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - tasks: - # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - - name: Add temporary IP on DEV1 idempotent treat File exists as OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 - and ('File exists' not in (add_ip.stdout | default(''))) - and ('File exists' not in (add_ip.stderr | default(''))) - - - name: Debug add ip results - ansible.builtin.debug: - msg: - - "add_ip.rc={{ add_ip.rc | default('') }}" - - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" - - # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb - ansible.builtin.raw: > - {{ pathprefix }} - bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 - register: dev2_mac_scan - changed_when: false - - - name: Set dev2_mac from fdb scan (if any) - ansible.builtin.set_fact: - dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" - changed_when: false - - - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true - register: dev2_arp_del - changed_when: false - failed_when: false - - - name: Add static ARP for DEV2 on DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent - register: dev2_arp_add - changed_when: dev2_arp_add.rc == 0 - failed_when: > - dev2_arp_add.rc != 0 - and ('File exists' not in (dev2_arp_add.stdout | default(''))) - and ('File exists' not in (dev2_arp_add.stderr | default(''))) - - - name: Debug static ARP result - ansible.builtin.debug: - msg: - - "dev2_mac={{ dev2_mac | default('UNSET') }}" - - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - - - name: Skip ARP add because MAC not found - when: dev2_mac is not defined or dev2_mac | length == 0 - ansible.builtin.debug: - msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" - - # ---------------------------- Local tunnel preparation ---------------------------- - - name: Pick a free local port for the tunnel controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - pick() { - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n 1)" - if command -v ss >/dev/null 2>&1; then - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then - echo "$p"; return 0 - fi - else - if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then - echo "$p"; return 0 - fi - fi - done - return 1 - } - pick - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control directory - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Debug chosen tunnel params - delegate_to: localhost - ansible.builtin.debug: - msg: - - "local_port={{ _local_port }}" - - "ctrl_dir={{ _ctrl_dir }}" - - "ctrl_sock={{ _ctrl_sock }}" - - - name: issue arping with own IP - ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 - - # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - - name: Start SSH tunnel via DEV1 background ControlMaster - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ dev1_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - "${USER}@${HOST}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running proper O check with destination - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - - name: Sanity show presence and permissions of passfiles on controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" - register: dev2_ls - changed_when: false - - - name: Debug passfiles listing - delegate_to: localhost - ansible.builtin.debug: - msg: - - "{{ (dev2_ls.stdout | default('')) | trim }}" - - "{{ (dev2_ls.stderr | default('')) | trim }}" - - # ---------------------------- Pick DEV2 password root only ---------------------------- - - name: Try DEV2 login with basicpass root - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass - changed_when: false - ignore_errors: true - - - name: Select basicpass as working passfile - when: dev2_try_basicpass.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass" - changed_when: false - - - name: Try DEV2 login with basicpass2 root only if first failed - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass2 ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass2 - changed_when: false - ignore_errors: true - - - name: Select basicpass2 as working passfile - when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass2" - changed_when: false - - - name: Set dev2_passfile_used to NONE if neither worked - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "NONE" - changed_when: false - - - name: Debug selected DEV2 passfile or NONE - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - - - name: Read DEV2 hostname via tunnel (busybox-safe) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" - args: - executable: /bin/bash - register: dev2_host_read - changed_when: false - - - name: Normalize hostnames for strict compare - ansible.builtin.set_fact: - _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" - _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - - - name: Debug hostname bytes (hex) - delegate_to: localhost - ansible.builtin.debug: - msg: - - "inv={{ _inv_hn | tojson }}" - - "dev1={{ _dev1_hn | tojson }}" - - "dev2={{ _dev2_hn | tojson }}" - - # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - - name: Enforce DEV2 hostname equals inventory and DEV1 - block: - - name: Fail if DEV2 hostname differs from inventory/DEV1 - ansible.builtin.fail: - msg: > - Hostname mismatch: DEV2='{{ _dev2_hn }}', - inventory='{{ _inv_hn }}', - DEV1='{{ _dev1_hn }}' - when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) - rescue: - - name: Initialize journal array for hostname mismatch - ansible.builtin.set_fact: - _journal: [] - _blocked: true - _prep_blocked: false - delegate_to: localhost - - - name: Append hostname mismatch to journal - ansible.builtin.set_fact: - _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" - delegate_to: localhost - - - name: Build control queue payload for indoor aborted journal (hostname) - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue (hostname) - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_hn_resp - changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_hn_resp.status != 200) or - (rmq_journal_indoor_aborted_hn_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) - delegate_to: localhost - - - name: Stop host after hostname mismatch - ansible.builtin.meta: end_host - - # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - - name: Initialize journal and flags - ansible.builtin.set_fact: - _journal: [] - _prep_blocked: false - _blocked: false - - - name: Build specific marker filename (no .bin) - ansible.builtin.set_fact: - _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - - - name: Check for any preparation markers on DEV2 (wildcard count) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "ls /tmp/prepared_for* 2>/dev/null | wc -l" - args: - executable: /bin/bash - register: dev2_prep_count - changed_when: false - ignore_errors: true - - - name: Soft-block if any prep markers already present - when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - - # ---------------------------- DEV2 version firmux primary check ---------------------------- - - name: Read DEV2 firmux release - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /usr/lib/release/firmux 2>/dev/null || true" - args: - executable: /bin/bash - register: dev2_firmux - changed_when: false - ignore_errors: true - - - name: Debug firmux content DEV2 - when: dev2_firmux is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - # ---------------------------- Normalize rebootin early ---------------------------- - - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" - _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" - _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" - _reboot_minutes: >- - {{ - 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) - else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) - }} - - - name: "Normalize rebootin (phase 2: requested flag)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" - - # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - - name: Build control queue payload for indoor start journal - ansible.builtin.set_fact: - journal_indoor_start: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 is reachable, starting with update. - image={{ image_filename }}, md5={{ image_md5 }}, - firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, - reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }} - when: dev2_passfile_used != "NONE" - delegate_to: localhost - - - name: Publish indoor start journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_start | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_start_resp - changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_start_resp.status != 200) or - (rmq_journal_indoor_start_resp.json is not defined) or - (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) - when: journal_indoor_start is defined - delegate_to: localhost - - # ---------------------------- Stage image on DEV2 no write ---------------------------- - - name: Check local presence of image file - delegate_to: localhost - ansible.builtin.stat: - path: "{{ image_filename }}" - register: local_img - - - name: Soft-block if image missing locally - when: not local_img.stat.exists - ansible.builtin.set_fact: - _blocked: true - _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - - - name: Compute local md5 of the image - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.shell: | - set -e - md5sum "{{ image_filename }}" | awk '{print $1}' - register: local_md5 - changed_when: false - - - name: Verify local md5 matches expected - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.assert: - that: - - (local_md5.stdout | trim) == image_md5 - fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" - success_msg: "Local md5 matches expected" - - # Optional SHA256 local - - name: Compute local sha256 of the image (if provided) - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sha256sum "{{ image_filename }}" | awk '{print $1}' - register: local_sha256 - changed_when: false - ignore_errors: true - - - name: Soft-block if local sha256 mismatch or unavailable - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - # fw_printenv health before upload (soft-fail) - - name: Read fw_printenv size (line count) on DEV2 - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "(fw_printenv 2>/dev/null | wc -l) || echo 0" - args: - executable: /bin/bash - register: dev2_fwenv_wc - changed_when: false - ignore_errors: true - - - name: Soft-block if fw_printenv too small - when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - - - name: Check existing image md5 on DEV2 NOFILE if absent - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_before - changed_when: false - - - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" scp \ - -P "$PORT" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" - args: - executable: /bin/bash - register: scp_push - changed_when: true - - - name: Compute md5 of image on DEV2 after copy or if exists - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_after - changed_when: false - - - name: Verify DEV2 md5 matches expected (soft-fail journal) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" - - # Optional SHA256 remote - - name: Compute sha256 of image on DEV2 (if provided) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" - args: - executable: /bin/bash - register: dev2_sha256_after - changed_when: false - ignore_errors: true - - - name: Soft-block if DEV2 sha256 mismatch/unavailable - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - - name: Run noninvasive update check on DEV2 update c - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -c '{{ dev2_image_path }}' 2>&1 || true" - args: - executable: /bin/bash - register: dev2_update_check - changed_when: false - - - name: Debug update c output DEV2 - when: dev2_update_check is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - - - name: Soft-block if update -c did not say valid - when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" - - # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - - name: Build control queue payload for indoor aborted journal - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - when: (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_resp - changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_resp.status != 200) or - (rmq_journal_indoor_aborted_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) - when: journal_indoor_aborted is defined - delegate_to: localhost - - # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - - name: Upgrade write and flip block - when: not (_blocked | default(false)) - block: - - name: Write image on DEV2 (this will take a while) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') - - - name: Read current active partition on DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_before - changed_when: false - failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - - - name: Determine new active value for DEV2 - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) - ansible.builtin.set_fact: - dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - - - name: Switch active partition to {{ dev2_new_active }} - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_setenv active {{ dev2_new_active }}" - args: - executable: /bin/bash - register: dev2_setenv_out - changed_when: true - - - name: Verify active partition flipped on DEV2 - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_after - changed_when: false - failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - - - name: Create specific prep marker on DEV2 (no generic) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "touch '{{ _marker_specific }}'" - args: - executable: /bin/bash - register: dev2_marker_write - changed_when: true - ignore_errors: true - - # ---------------------------- Reboot scheduling (normalized) ---------------------------- - - name: Schedule delayed reboot on DEV2 minutes - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - SECS="$(( {{ _reboot_minutes | int }} * 60 ))" - CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" - args: - executable: /bin/bash - register: dev2_reboot_sched - changed_when: true - ignore_errors: true - - - name: Build control queue payload for indoor-restart-scheduled tag - ansible.builtin.set_fact: - tag_restart_sched_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_add" - task_result: "indoor-restart-scheduled" - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_restart_sched_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_restart_sched_resp - changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_tag_restart_sched_resp.status != 200) or - (rmq_tag_restart_sched_resp.json is not defined) or - (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) - when: tag_restart_sched_payload is defined - delegate_to: localhost - - - - name: Note reboot request bad format - when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested - ansible.builtin.debug: - msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" - - # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - - name: Build control queue payload for indoor updated journal - ansible.builtin.set_fact: - _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" - when: not (_blocked | default(false)) - delegate_to: localhost - - - name: Build indoor updated journal payload text - ansible.builtin.set_fact: - journal_indoor_updated: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 updated, scheduling reboot {{ - 'now' if (_reboot_requested and (_reboot_minutes | int) == 0) - else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) - else 'not requested' - }}. - write_done={{ _write_success }}, - active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, - active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Publish indoor updated journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_updated | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_updated_resp - changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_updated_resp.status != 200) or - (rmq_journal_indoor_updated_resp.json is not defined) or - (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) - when: journal_indoor_updated is defined - delegate_to: localhost - - - - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) - ansible.builtin.set_fact: - au_attempt: 1 - au_max_attempts: 3 - au_delay_sec: "{{ ((_reboot_minutes | int) * 60) + 360 }}" - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Generate correlation ID and original emitted timestamp - ansible.builtin.set_fact: - au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" - au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" - when: au_delay_sec is defined - delegate_to: localhost - - - name: Build after-upgrade check payload (attempt 1) - ansible.builtin.set_fact: - afterupgrade_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" - attempt: "{{ au_attempt }}" - max_attempts: "{{ au_max_attempts }}" - current_delay_sec: "{{ au_delay_sec }}" - correlation_id: "{{ au_correlation_id }}" - original_emitted_at: "{{ au_original_emitted_at }}" - schema_version: 1 - when: au_delay_sec is defined - delegate_to: localhost - - - name: Debug x-delay about to be sent (ms) - ansible.builtin.debug: - msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" - when: afterupgrade_payload is defined - delegate_to: localhost - - - name: Publish delayed after-upgrade check to holding exchange - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - headers: - x-delay: "{{ (au_delay_sec | int) * 1000 }}" - routing_key: "{{ afterupgrade_routing_key }}" - payload: "{{ afterupgrade_payload | to_json }}" - payload_encoding: "string" - register: rmq_afterupgrade_resp - changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_afterupgrade_resp.status != 200) or - (rmq_afterupgrade_resp.json is not defined) - when: afterupgrade_payload is defined - delegate_to: localhost - - - # ---------------------------- summary ---------------------------- - - name: Summary show key results - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - "local_image_present={{ local_img.stat.exists | default(false) }}" - - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - - "reboot_requested={{ _reboot_requested | default(false) }}" - - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - - "prep_blocked={{ _prep_blocked | default(false) }}" - - "blocked={{ _blocked | default(false) }}" - - "journal={{ (_journal | default([])) | join(' || ') }}" - - post_tasks: - - name: Cleanup always - block: - - ansible.builtin.debug: - msg: "entering cleanup block" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel best effort - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir best effort - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - changed_when: del_ip.rc == 0 - failed_when: > - del_ip.rc != 0 - and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) - and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - - - name: Debug del ip results - ansible.builtin.debug: - msg: - - "del_ip.rc={{ del_ip.rc | default('') }}" - - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" - when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-day2 b/files/ansible-playbooks/update-indoor.yml-day2 deleted file mode 100644 index 3d81c6e..0000000 --- a/files/ansible-playbooks/update-indoor.yml-day2 +++ /dev/null @@ -1,1117 +0,0 @@ ---- -# update-secondline.yml -- name: Second-line upgrade tunneled noninvasive engine - hosts: all - gather_facts: no - - vars: - # Busybox-safe PATH prefix for all remote raw calls on DEV1 - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # DEV1 credentials (stable, like rebootin222) - dev1_user: "root" - dev1_pass: "wavewave" - - # Tunnel target DEV2 behind DEV1 - dev2_host: "192.168.1.1" - dev2_port: 22 - - # Temp IP we add to DEV1 so it can reach DEV2 - dev2_side_ip: "192.168.1.11/24" - dev1_iface: "br-wan" - - # DEV2 behind the tunnel - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # SSH options used from controller - ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" - - # Image to stage on DEV2 noninvasive we do not run update -w here - image_filename: "fox200-2.2.1-r6801.bin" - image_md5: "56b7211709de617e058b98d4204e2562" - # Optional SHA256; leave empty to skip SHA256 checks - image_sha256: "" - dev2_image_dir: "/tmp" - dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" - - # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). - rebootin: "" - - # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" - - - pre_tasks: - # ------------------------------- Hostname sanity DEV1 ------------------------------- - - name: Read DEV1 hostname from env busybox safe - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 connected hostname differs from inventory - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - tasks: - # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - - name: Add temporary IP on DEV1 idempotent treat File exists as OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 - and ('File exists' not in (add_ip.stdout | default(''))) - and ('File exists' not in (add_ip.stderr | default(''))) - - - name: Debug add ip results - ansible.builtin.debug: - msg: - - "add_ip.rc={{ add_ip.rc | default('') }}" - - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" - - # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb - ansible.builtin.raw: > - {{ pathprefix }} - bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 - register: dev2_mac_scan - changed_when: false - - - name: Set dev2_mac from fdb scan (if any) - ansible.builtin.set_fact: - dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" - changed_when: false - - - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true - register: dev2_arp_del - changed_when: false - failed_when: false - - - name: Add static ARP for DEV2 on DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent - register: dev2_arp_add - changed_when: dev2_arp_add.rc == 0 - failed_when: > - dev2_arp_add.rc != 0 - and ('File exists' not in (dev2_arp_add.stdout | default(''))) - and ('File exists' not in (dev2_arp_add.stderr | default(''))) - - - name: Debug static ARP result - ansible.builtin.debug: - msg: - - "dev2_mac={{ dev2_mac | default('UNSET') }}" - - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - - - name: Skip ARP add because MAC not found - when: dev2_mac is not defined or dev2_mac | length == 0 - ansible.builtin.debug: - msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" - - # ---------------------------- Local tunnel preparation ---------------------------- - - name: Pick a free local port for the tunnel controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - pick() { - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n 1)" - if command -v ss >/dev/null 2>&1; then - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then - echo "$p"; return 0 - fi - else - if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then - echo "$p"; return 0 - fi - fi - done - return 1 - } - pick - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control directory - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Debug chosen tunnel params - delegate_to: localhost - ansible.builtin.debug: - msg: - - "local_port={{ _local_port }}" - - "ctrl_dir={{ _ctrl_dir }}" - - "ctrl_sock={{ _ctrl_sock }}" - - - name: issue arping with own IP - ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 - - # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - - name: Start SSH tunnel via DEV1 background ControlMaster - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ dev1_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - "${USER}@${HOST}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running proper O check with destination - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - - name: Sanity show presence and permissions of passfiles on controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" - register: dev2_ls - changed_when: false - - - name: Debug passfiles listing - delegate_to: localhost - ansible.builtin.debug: - msg: - - "{{ (dev2_ls.stdout | default('')) | trim }}" - - "{{ (dev2_ls.stderr | default('')) | trim }}" - - # ---------------------------- Pick DEV2 password root only ---------------------------- - - name: Try DEV2 login with basicpass root - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass - changed_when: false - ignore_errors: true - - - name: Select basicpass as working passfile - when: dev2_try_basicpass.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass" - changed_when: false - - - name: Try DEV2 login with basicpass2 root only if first failed - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass2 ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass2 - changed_when: false - ignore_errors: true - - - name: Select basicpass2 as working passfile - when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass2" - changed_when: false - - - name: Set dev2_passfile_used to NONE if neither worked - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "NONE" - changed_when: false - - - name: Debug selected DEV2 passfile or NONE - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - - - name: Read DEV2 hostname via tunnel (busybox-safe) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" - args: - executable: /bin/bash - register: dev2_host_read - changed_when: false - - - name: Normalize hostnames for strict compare - ansible.builtin.set_fact: - _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" - _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - - - name: Debug hostname bytes (hex) - delegate_to: localhost - ansible.builtin.debug: - msg: - - "inv={{ _inv_hn | tojson }}" - - "dev1={{ _dev1_hn | tojson }}" - - "dev2={{ _dev2_hn | tojson }}" - - # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - - name: Enforce DEV2 hostname equals inventory and DEV1 - block: - - name: Fail if DEV2 hostname differs from inventory/DEV1 - ansible.builtin.fail: - msg: > - Hostname mismatch: DEV2='{{ _dev2_hn }}', - inventory='{{ _inv_hn }}', - DEV1='{{ _dev1_hn }}' - when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) - rescue: - - name: Initialize journal array for hostname mismatch - ansible.builtin.set_fact: - _journal: [] - _blocked: true - _prep_blocked: false - delegate_to: localhost - - - name: Append hostname mismatch to journal - ansible.builtin.set_fact: - _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" - delegate_to: localhost - - - name: Build control queue payload for indoor aborted journal (hostname) - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue (hostname) - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_hn_resp - changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_hn_resp.status != 200) or - (rmq_journal_indoor_aborted_hn_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) - delegate_to: localhost - - - name: Stop host after hostname mismatch - ansible.builtin.meta: end_host - - # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - - name: Initialize journal and flags - ansible.builtin.set_fact: - _journal: [] - _prep_blocked: false - _blocked: false - - - name: Build specific marker filename (no .bin) - ansible.builtin.set_fact: - _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - - - name: Check for any preparation markers on DEV2 (wildcard count) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "ls /tmp/prepared_for* 2>/dev/null | wc -l" - args: - executable: /bin/bash - register: dev2_prep_count - changed_when: false - ignore_errors: true - - - name: Soft-block if any prep markers already present - when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - - # ---------------------------- DEV2 version firmux primary check ---------------------------- - - name: Read DEV2 firmux release - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /usr/lib/release/firmux 2>/dev/null || true" - args: - executable: /bin/bash - register: dev2_firmux - changed_when: false - ignore_errors: true - - - name: Debug firmux content DEV2 - when: dev2_firmux is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- - - name: "Normalize rebootin (phase 1: raw/is_now/is_int/HOURS)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" - _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" - _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" - _reboot_hours: >- - {{ - 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) - else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) - }} - - - name: "Normalize rebootin (phase 2: requested flag)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" - - # Compute effective seconds from HOURS (0h → ~20 seconds grace); clamp non-negative - - name: "Normalize rebootin (phase 3: compute seconds from HOURS)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_seconds: >- - {{ - 20 - if (_reboot_requested | default(false)) and ((_reboot_hours | int) <= 0) - else - (((_reboot_hours | int) * 3600)) - if (_reboot_requested | default(false)) - else 0 - }} - - # Derive minutes from seconds (non-negative), for tasks/summary that expect minutes - - name: "Normalize rebootin (phase 4: derive minutes)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_minutes: "{{ ((_reboot_seconds | int) // 60) if (_reboot_seconds | int) > 0 else 0 }}" - - # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - - name: Build control queue payload for indoor start journal - ansible.builtin.set_fact: - journal_indoor_start: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 is reachable, starting with update. - image={{ image_filename }}, md5={{ image_md5 }}, - firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, - reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} - when: dev2_passfile_used != "NONE" - delegate_to: localhost - - - name: Publish indoor start journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_start | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_start_resp - changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_start_resp.status != 200) or - (rmq_journal_indoor_start_resp.json is not defined) or - (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) - when: journal_indoor_start is defined - delegate_to: localhost - - # ---------------------------- Stage image on DEV2 no write ---------------------------- - - name: Check local presence of image file - delegate_to: localhost - ansible.builtin.stat: - path: "{{ image_filename }}" - register: local_img - - - name: Soft-block if image missing locally - when: not local_img.stat.exists - ansible.builtin.set_fact: - _blocked: true - _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - - - name: Compute local md5 of the image - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.shell: | - set -e - md5sum "{{ image_filename }}" | awk '{print $1}' - register: local_md5 - changed_when: false - - - name: Verify local md5 matches expected - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.assert: - that: - - (local_md5.stdout | trim) == image_md5 - fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" - success_msg: "Local md5 matches expected" - - # Optional SHA256 local - - name: Compute local sha256 of the image (if provided) - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sha256sum "{{ image_filename }}" | awk '{print $1}' - register: local_sha256 - changed_when: false - ignore_errors: true - - - name: Soft-block if local sha256 mismatch or unavailable - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - # fw_printenv health before upload (soft-fail) - - name: Read fw_printenv size (line count) on DEV2 - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "(fw_printenv 2>/dev/null | wc -l) || echo 0" - args: - executable: /bin/bash - register: dev2_fwenv_wc - changed_when: false - ignore_errors: true - - - name: Soft-block if fw_printenv too small - when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - - - name: Check existing image md5 on DEV2 NOFILE if absent - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_before - changed_when: false - - - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" scp \ - -P "$PORT" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" - args: - executable: /bin/bash - register: scp_push - changed_when: true - - - name: Compute md5 of image on DEV2 after copy or if exists - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_after - changed_when: false - - - name: Verify DEV2 md5 matches expected (soft-fail journal) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" - - # Optional SHA256 remote - - name: Compute sha256 of image on DEV2 (if provided) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" - args: - executable: /bin/bash - register: dev2_sha256_after - changed_when: false - ignore_errors: true - - - name: Soft-block if DEV2 sha256 mismatch/unavailable - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - - name: Run noninvasive update check on DEV2 update c - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -c '{{ dev2_image_path }}' 2>&1 || true" - args: - executable: /bin/bash - register: dev2_update_check - changed_when: false - - - name: Debug update c output DEV2 - when: dev2_update_check is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - - - name: Soft-block if update -c did not say valid - when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" - - # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - - name: Build control queue payload for indoor aborted journal - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - when: (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_resp - changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_resp.status != 200) or - (rmq_journal_indoor_aborted_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) - when: journal_indoor_aborted is defined - delegate_to: localhost - - # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - - name: Upgrade write and flip block - when: not (_blocked | default(false)) - block: - - name: Write image on DEV2 (this will take a while) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') - - - name: Read current active partition on DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_before - changed_when: false - failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - - - name: Determine new active value for DEV2 - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) - ansible.builtin.set_fact: - dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - - - name: Switch active partition to {{ dev2_new_active }} - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_setenv active {{ dev2_new_active }}" - args: - executable: /bin/bash - register: dev2_setenv_out - changed_when: true - - - name: Verify active partition flipped on DEV2 - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_after - changed_when: false - failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - - - name: Create specific prep marker on DEV2 (no generic) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "touch '{{ _marker_specific }}'" - args: - executable: /bin/bash - register: dev2_marker_write - changed_when: true - ignore_errors: true - - # ---------------------------- Reboot scheduling (normalized) ---------------------------- - - name: Schedule delayed reboot on DEV2 minutes - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - SECS="{{ _reboot_seconds | int }}" - CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" - args: - executable: /bin/bash - register: dev2_reboot_sched - changed_when: true - ignore_errors: true - - - name: Build control queue payload for indoor-restart-scheduled tag - ansible.builtin.set_fact: - tag_restart_sched_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_add" - task_result: "indoor-restart-scheduled" - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_restart_sched_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_restart_sched_resp - changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_tag_restart_sched_resp.status != 200) or - (rmq_tag_restart_sched_resp.json is not defined) or - (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) - when: tag_restart_sched_payload is defined - delegate_to: localhost - - - - name: Note reboot request bad format - when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested - ansible.builtin.debug: - msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" - - # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - - name: Build control queue payload for indoor updated journal - ansible.builtin.set_fact: - _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" - when: not (_blocked | default(false)) - delegate_to: localhost - - - name: Build indoor updated journal payload text - ansible.builtin.set_fact: - journal_indoor_updated: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 updated, scheduling reboot {{ - ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) - else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) - else 'not requested' - }}. - write_done={{ _write_success }}, - active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, - active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Publish indoor updated journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_updated | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_updated_resp - changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_updated_resp.status != 200) or - (rmq_journal_indoor_updated_resp.json is not defined) or - (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) - when: journal_indoor_updated is defined - delegate_to: localhost - - - - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) - ansible.builtin.set_fact: - au_attempt: 1 - au_max_attempts: 3 - au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Generate correlation ID and original emitted timestamp - ansible.builtin.set_fact: - au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" - au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" - when: au_delay_sec is defined - delegate_to: localhost - - - name: Build after-upgrade check payload (attempt 1) - ansible.builtin.set_fact: - afterupgrade_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" - attempt: "{{ au_attempt }}" - max_attempts: "{{ au_max_attempts }}" - current_delay_sec: "{{ au_delay_sec }}" - correlation_id: "{{ au_correlation_id }}" - original_emitted_at: "{{ au_original_emitted_at }}" - schema_version: 1 - when: au_delay_sec is defined - delegate_to: localhost - - - name: Debug x-delay about to be sent (ms) - ansible.builtin.debug: - msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" - when: afterupgrade_payload is defined - delegate_to: localhost - - - name: Publish delayed after-upgrade check to holding exchange - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - headers: - x-delay: "{{ (au_delay_sec | int) * 1000 }}" - routing_key: "{{ afterupgrade_routing_key }}" - payload: "{{ afterupgrade_payload | to_json }}" - payload_encoding: "string" - register: rmq_afterupgrade_resp - changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_afterupgrade_resp.status != 200) or - (rmq_afterupgrade_resp.json is not defined) - when: afterupgrade_payload is defined - delegate_to: localhost - - - # ---------------------------- summary ---------------------------- - - name: Summary show key results - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - "local_image_present={{ local_img.stat.exists | default(false) }}" - - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - - "reboot_requested={{ _reboot_requested | default(false) }}" - - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - - "prep_blocked={{ _prep_blocked | default(false) }}" - - "blocked={{ _blocked | default(false) }}" - - "journal={{ (_journal | default([])) | join(' || ') }}" - - post_tasks: - - name: Cleanup always - block: - - ansible.builtin.debug: - msg: "entering cleanup block" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel best effort - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir best effort - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - changed_when: del_ip.rc == 0 - failed_when: > - del_ip.rc != 0 - and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) - and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - - - name: Debug del ip results - ansible.builtin.debug: - msg: - - "del_ip.rc={{ del_ip.rc | default('') }}" - - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" - when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-fixingerrors b/files/ansible-playbooks/update-indoor.yml-fixingerrors deleted file mode 100644 index 3d81c6e..0000000 --- a/files/ansible-playbooks/update-indoor.yml-fixingerrors +++ /dev/null @@ -1,1117 +0,0 @@ ---- -# update-secondline.yml -- name: Second-line upgrade tunneled noninvasive engine - hosts: all - gather_facts: no - - vars: - # Busybox-safe PATH prefix for all remote raw calls on DEV1 - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # DEV1 credentials (stable, like rebootin222) - dev1_user: "root" - dev1_pass: "wavewave" - - # Tunnel target DEV2 behind DEV1 - dev2_host: "192.168.1.1" - dev2_port: 22 - - # Temp IP we add to DEV1 so it can reach DEV2 - dev2_side_ip: "192.168.1.11/24" - dev1_iface: "br-wan" - - # DEV2 behind the tunnel - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # SSH options used from controller - ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" - - # Image to stage on DEV2 noninvasive we do not run update -w here - image_filename: "fox200-2.2.1-r6801.bin" - image_md5: "56b7211709de617e058b98d4204e2562" - # Optional SHA256; leave empty to skip SHA256 checks - image_sha256: "" - dev2_image_dir: "/tmp" - dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" - - # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). - rebootin: "" - - # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" - - - pre_tasks: - # ------------------------------- Hostname sanity DEV1 ------------------------------- - - name: Read DEV1 hostname from env busybox safe - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 connected hostname differs from inventory - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - tasks: - # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - - name: Add temporary IP on DEV1 idempotent treat File exists as OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 - and ('File exists' not in (add_ip.stdout | default(''))) - and ('File exists' not in (add_ip.stderr | default(''))) - - - name: Debug add ip results - ansible.builtin.debug: - msg: - - "add_ip.rc={{ add_ip.rc | default('') }}" - - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" - - # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb - ansible.builtin.raw: > - {{ pathprefix }} - bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 - register: dev2_mac_scan - changed_when: false - - - name: Set dev2_mac from fdb scan (if any) - ansible.builtin.set_fact: - dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" - changed_when: false - - - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true - register: dev2_arp_del - changed_when: false - failed_when: false - - - name: Add static ARP for DEV2 on DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent - register: dev2_arp_add - changed_when: dev2_arp_add.rc == 0 - failed_when: > - dev2_arp_add.rc != 0 - and ('File exists' not in (dev2_arp_add.stdout | default(''))) - and ('File exists' not in (dev2_arp_add.stderr | default(''))) - - - name: Debug static ARP result - ansible.builtin.debug: - msg: - - "dev2_mac={{ dev2_mac | default('UNSET') }}" - - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - - - name: Skip ARP add because MAC not found - when: dev2_mac is not defined or dev2_mac | length == 0 - ansible.builtin.debug: - msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" - - # ---------------------------- Local tunnel preparation ---------------------------- - - name: Pick a free local port for the tunnel controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - pick() { - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n 1)" - if command -v ss >/dev/null 2>&1; then - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then - echo "$p"; return 0 - fi - else - if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then - echo "$p"; return 0 - fi - fi - done - return 1 - } - pick - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control directory - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Debug chosen tunnel params - delegate_to: localhost - ansible.builtin.debug: - msg: - - "local_port={{ _local_port }}" - - "ctrl_dir={{ _ctrl_dir }}" - - "ctrl_sock={{ _ctrl_sock }}" - - - name: issue arping with own IP - ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 - - # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - - name: Start SSH tunnel via DEV1 background ControlMaster - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ dev1_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - "${USER}@${HOST}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running proper O check with destination - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - - name: Sanity show presence and permissions of passfiles on controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" - register: dev2_ls - changed_when: false - - - name: Debug passfiles listing - delegate_to: localhost - ansible.builtin.debug: - msg: - - "{{ (dev2_ls.stdout | default('')) | trim }}" - - "{{ (dev2_ls.stderr | default('')) | trim }}" - - # ---------------------------- Pick DEV2 password root only ---------------------------- - - name: Try DEV2 login with basicpass root - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass - changed_when: false - ignore_errors: true - - - name: Select basicpass as working passfile - when: dev2_try_basicpass.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass" - changed_when: false - - - name: Try DEV2 login with basicpass2 root only if first failed - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass2 ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass2 - changed_when: false - ignore_errors: true - - - name: Select basicpass2 as working passfile - when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass2" - changed_when: false - - - name: Set dev2_passfile_used to NONE if neither worked - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "NONE" - changed_when: false - - - name: Debug selected DEV2 passfile or NONE - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - - - name: Read DEV2 hostname via tunnel (busybox-safe) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" - args: - executable: /bin/bash - register: dev2_host_read - changed_when: false - - - name: Normalize hostnames for strict compare - ansible.builtin.set_fact: - _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" - _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - - - name: Debug hostname bytes (hex) - delegate_to: localhost - ansible.builtin.debug: - msg: - - "inv={{ _inv_hn | tojson }}" - - "dev1={{ _dev1_hn | tojson }}" - - "dev2={{ _dev2_hn | tojson }}" - - # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - - name: Enforce DEV2 hostname equals inventory and DEV1 - block: - - name: Fail if DEV2 hostname differs from inventory/DEV1 - ansible.builtin.fail: - msg: > - Hostname mismatch: DEV2='{{ _dev2_hn }}', - inventory='{{ _inv_hn }}', - DEV1='{{ _dev1_hn }}' - when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) - rescue: - - name: Initialize journal array for hostname mismatch - ansible.builtin.set_fact: - _journal: [] - _blocked: true - _prep_blocked: false - delegate_to: localhost - - - name: Append hostname mismatch to journal - ansible.builtin.set_fact: - _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" - delegate_to: localhost - - - name: Build control queue payload for indoor aborted journal (hostname) - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue (hostname) - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_hn_resp - changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_hn_resp.status != 200) or - (rmq_journal_indoor_aborted_hn_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) - delegate_to: localhost - - - name: Stop host after hostname mismatch - ansible.builtin.meta: end_host - - # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - - name: Initialize journal and flags - ansible.builtin.set_fact: - _journal: [] - _prep_blocked: false - _blocked: false - - - name: Build specific marker filename (no .bin) - ansible.builtin.set_fact: - _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - - - name: Check for any preparation markers on DEV2 (wildcard count) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "ls /tmp/prepared_for* 2>/dev/null | wc -l" - args: - executable: /bin/bash - register: dev2_prep_count - changed_when: false - ignore_errors: true - - - name: Soft-block if any prep markers already present - when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - - # ---------------------------- DEV2 version firmux primary check ---------------------------- - - name: Read DEV2 firmux release - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /usr/lib/release/firmux 2>/dev/null || true" - args: - executable: /bin/bash - register: dev2_firmux - changed_when: false - ignore_errors: true - - - name: Debug firmux content DEV2 - when: dev2_firmux is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- - - name: "Normalize rebootin (phase 1: raw/is_now/is_int/HOURS)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" - _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" - _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" - _reboot_hours: >- - {{ - 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) - else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) - }} - - - name: "Normalize rebootin (phase 2: requested flag)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" - - # Compute effective seconds from HOURS (0h → ~20 seconds grace); clamp non-negative - - name: "Normalize rebootin (phase 3: compute seconds from HOURS)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_seconds: >- - {{ - 20 - if (_reboot_requested | default(false)) and ((_reboot_hours | int) <= 0) - else - (((_reboot_hours | int) * 3600)) - if (_reboot_requested | default(false)) - else 0 - }} - - # Derive minutes from seconds (non-negative), for tasks/summary that expect minutes - - name: "Normalize rebootin (phase 4: derive minutes)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_minutes: "{{ ((_reboot_seconds | int) // 60) if (_reboot_seconds | int) > 0 else 0 }}" - - # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - - name: Build control queue payload for indoor start journal - ansible.builtin.set_fact: - journal_indoor_start: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 is reachable, starting with update. - image={{ image_filename }}, md5={{ image_md5 }}, - firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, - reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} - when: dev2_passfile_used != "NONE" - delegate_to: localhost - - - name: Publish indoor start journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_start | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_start_resp - changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_start_resp.status != 200) or - (rmq_journal_indoor_start_resp.json is not defined) or - (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) - when: journal_indoor_start is defined - delegate_to: localhost - - # ---------------------------- Stage image on DEV2 no write ---------------------------- - - name: Check local presence of image file - delegate_to: localhost - ansible.builtin.stat: - path: "{{ image_filename }}" - register: local_img - - - name: Soft-block if image missing locally - when: not local_img.stat.exists - ansible.builtin.set_fact: - _blocked: true - _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - - - name: Compute local md5 of the image - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.shell: | - set -e - md5sum "{{ image_filename }}" | awk '{print $1}' - register: local_md5 - changed_when: false - - - name: Verify local md5 matches expected - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.assert: - that: - - (local_md5.stdout | trim) == image_md5 - fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" - success_msg: "Local md5 matches expected" - - # Optional SHA256 local - - name: Compute local sha256 of the image (if provided) - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sha256sum "{{ image_filename }}" | awk '{print $1}' - register: local_sha256 - changed_when: false - ignore_errors: true - - - name: Soft-block if local sha256 mismatch or unavailable - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - # fw_printenv health before upload (soft-fail) - - name: Read fw_printenv size (line count) on DEV2 - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "(fw_printenv 2>/dev/null | wc -l) || echo 0" - args: - executable: /bin/bash - register: dev2_fwenv_wc - changed_when: false - ignore_errors: true - - - name: Soft-block if fw_printenv too small - when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - - - name: Check existing image md5 on DEV2 NOFILE if absent - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_before - changed_when: false - - - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" scp \ - -P "$PORT" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" - args: - executable: /bin/bash - register: scp_push - changed_when: true - - - name: Compute md5 of image on DEV2 after copy or if exists - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_after - changed_when: false - - - name: Verify DEV2 md5 matches expected (soft-fail journal) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" - - # Optional SHA256 remote - - name: Compute sha256 of image on DEV2 (if provided) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" - args: - executable: /bin/bash - register: dev2_sha256_after - changed_when: false - ignore_errors: true - - - name: Soft-block if DEV2 sha256 mismatch/unavailable - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - - name: Run noninvasive update check on DEV2 update c - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -c '{{ dev2_image_path }}' 2>&1 || true" - args: - executable: /bin/bash - register: dev2_update_check - changed_when: false - - - name: Debug update c output DEV2 - when: dev2_update_check is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - - - name: Soft-block if update -c did not say valid - when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" - - # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - - name: Build control queue payload for indoor aborted journal - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - when: (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_resp - changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_resp.status != 200) or - (rmq_journal_indoor_aborted_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) - when: journal_indoor_aborted is defined - delegate_to: localhost - - # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - - name: Upgrade write and flip block - when: not (_blocked | default(false)) - block: - - name: Write image on DEV2 (this will take a while) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') - - - name: Read current active partition on DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_before - changed_when: false - failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - - - name: Determine new active value for DEV2 - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) - ansible.builtin.set_fact: - dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - - - name: Switch active partition to {{ dev2_new_active }} - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_setenv active {{ dev2_new_active }}" - args: - executable: /bin/bash - register: dev2_setenv_out - changed_when: true - - - name: Verify active partition flipped on DEV2 - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_after - changed_when: false - failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - - - name: Create specific prep marker on DEV2 (no generic) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "touch '{{ _marker_specific }}'" - args: - executable: /bin/bash - register: dev2_marker_write - changed_when: true - ignore_errors: true - - # ---------------------------- Reboot scheduling (normalized) ---------------------------- - - name: Schedule delayed reboot on DEV2 minutes - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - SECS="{{ _reboot_seconds | int }}" - CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" - args: - executable: /bin/bash - register: dev2_reboot_sched - changed_when: true - ignore_errors: true - - - name: Build control queue payload for indoor-restart-scheduled tag - ansible.builtin.set_fact: - tag_restart_sched_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_add" - task_result: "indoor-restart-scheduled" - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_restart_sched_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_restart_sched_resp - changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_tag_restart_sched_resp.status != 200) or - (rmq_tag_restart_sched_resp.json is not defined) or - (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) - when: tag_restart_sched_payload is defined - delegate_to: localhost - - - - name: Note reboot request bad format - when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested - ansible.builtin.debug: - msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" - - # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - - name: Build control queue payload for indoor updated journal - ansible.builtin.set_fact: - _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" - when: not (_blocked | default(false)) - delegate_to: localhost - - - name: Build indoor updated journal payload text - ansible.builtin.set_fact: - journal_indoor_updated: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 updated, scheduling reboot {{ - ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) - else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) - else 'not requested' - }}. - write_done={{ _write_success }}, - active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, - active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Publish indoor updated journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_updated | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_updated_resp - changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_updated_resp.status != 200) or - (rmq_journal_indoor_updated_resp.json is not defined) or - (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) - when: journal_indoor_updated is defined - delegate_to: localhost - - - - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) - ansible.builtin.set_fact: - au_attempt: 1 - au_max_attempts: 3 - au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Generate correlation ID and original emitted timestamp - ansible.builtin.set_fact: - au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" - au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" - when: au_delay_sec is defined - delegate_to: localhost - - - name: Build after-upgrade check payload (attempt 1) - ansible.builtin.set_fact: - afterupgrade_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" - attempt: "{{ au_attempt }}" - max_attempts: "{{ au_max_attempts }}" - current_delay_sec: "{{ au_delay_sec }}" - correlation_id: "{{ au_correlation_id }}" - original_emitted_at: "{{ au_original_emitted_at }}" - schema_version: 1 - when: au_delay_sec is defined - delegate_to: localhost - - - name: Debug x-delay about to be sent (ms) - ansible.builtin.debug: - msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" - when: afterupgrade_payload is defined - delegate_to: localhost - - - name: Publish delayed after-upgrade check to holding exchange - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - headers: - x-delay: "{{ (au_delay_sec | int) * 1000 }}" - routing_key: "{{ afterupgrade_routing_key }}" - payload: "{{ afterupgrade_payload | to_json }}" - payload_encoding: "string" - register: rmq_afterupgrade_resp - changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_afterupgrade_resp.status != 200) or - (rmq_afterupgrade_resp.json is not defined) - when: afterupgrade_payload is defined - delegate_to: localhost - - - # ---------------------------- summary ---------------------------- - - name: Summary show key results - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - "local_image_present={{ local_img.stat.exists | default(false) }}" - - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - - "reboot_requested={{ _reboot_requested | default(false) }}" - - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - - "prep_blocked={{ _prep_blocked | default(false) }}" - - "blocked={{ _blocked | default(false) }}" - - "journal={{ (_journal | default([])) | join(' || ') }}" - - post_tasks: - - name: Cleanup always - block: - - ansible.builtin.debug: - msg: "entering cleanup block" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel best effort - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir best effort - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - changed_when: del_ip.rc == 0 - failed_when: > - del_ip.rc != 0 - and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) - and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - - - name: Debug del ip results - ansible.builtin.debug: - msg: - - "del_ip.rc={{ del_ip.rc | default('') }}" - - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" - when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml2 b/files/ansible-playbooks/update-indoor.yml2 deleted file mode 100644 index 96643cb..0000000 --- a/files/ansible-playbooks/update-indoor.yml2 +++ /dev/null @@ -1,1108 +0,0 @@ ---- -# update-secondline.yml -- name: Second-line upgrade tunneled noninvasive engine - hosts: all - gather_facts: no - - vars: - # Busybox-safe PATH prefix for all remote raw calls on DEV1 - pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " - - # DEV1 credentials (stable, like rebootin222) - dev1_user: "root" - dev1_pass: "wavewave" - - # Tunnel target DEV2 behind DEV1 - dev2_host: "192.168.1.1" - dev2_port: 22 - - # Temp IP we add to DEV1 so it can reach DEV2 - dev2_side_ip: "192.168.1.11/24" - dev1_iface: "br-wan" - - # DEV2 behind the tunnel - dev2_ssh_user: "root" - dev2_passfiles: - - "basicpass" - - "basicpass2" - - # SSH options used from controller - ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" - - # Image to stage on DEV2 noninvasive we do not run update -w here - image_filename: "fox200-2.2.1-r6801.bin" - image_md5: "56b7211709de617e058b98d4204e2562" - # Optional SHA256; leave empty to skip SHA256 checks - image_sha256: "" - dev2_image_dir: "/tmp" - dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" - - # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 - rebootin: "" - - # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" - afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" - - - pre_tasks: - # ------------------------------- Hostname sanity DEV1 ------------------------------- - - name: Read DEV1 hostname from env busybox safe - ansible.builtin.raw: > - {{ pathprefix }} - (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") - register: dev1_host_read - changed_when: false - - - name: Stop if DEV1 connected hostname differs from inventory - ansible.builtin.meta: end_host - when: (dev1_host_read.stdout | trim | length > 0) and - ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) - - tasks: - # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - - name: Add temporary IP on DEV1 idempotent treat File exists as OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} - register: add_ip - changed_when: add_ip.rc == 0 - failed_when: > - add_ip.rc != 0 - and ('File exists' not in (add_ip.stdout | default(''))) - and ('File exists' not in (add_ip.stderr | default(''))) - - - name: Debug add ip results - ansible.builtin.debug: - msg: - - "add_ip.rc={{ add_ip.rc | default('') }}" - - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" - - # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb - ansible.builtin.raw: > - {{ pathprefix }} - bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 - register: dev2_mac_scan - changed_when: false - - - name: Set dev2_mac from fdb scan (if any) - ansible.builtin.set_fact: - dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" - changed_when: false - - - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true - register: dev2_arp_del - changed_when: false - failed_when: false - - - name: Add static ARP for DEV2 on DEV1 - ansible.builtin.raw: > - {{ pathprefix }} - ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent - register: dev2_arp_add - changed_when: dev2_arp_add.rc == 0 - failed_when: > - dev2_arp_add.rc != 0 - and ('File exists' not in (dev2_arp_add.stdout | default(''))) - and ('File exists' not in (dev2_arp_add.stderr | default(''))) - - - name: Debug static ARP result - ansible.builtin.debug: - msg: - - "dev2_mac={{ dev2_mac | default('UNSET') }}" - - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - - - name: Skip ARP add because MAC not found - when: dev2_mac is not defined or dev2_mac | length == 0 - ansible.builtin.debug: - msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" - - # ---------------------------- Local tunnel preparation ---------------------------- - - name: Pick a free local port for the tunnel controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - pick() { - for i in $(seq 1 25); do - p="$(shuf -i 20000-39999 -n 1)" - if command -v ss >/dev/null 2>&1; then - if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then - echo "$p"; return 0 - fi - else - if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then - echo "$p"; return 0 - fi - fi - done - return 1 - } - pick - register: pick_port - changed_when: false - - - name: Stop if no free local port - ansible.builtin.meta: end_host - when: (pick_port.stdout | trim | length) == 0 - - - name: Record chosen local port and control directory - delegate_to: localhost - ansible.builtin.set_fact: - _local_port: "{{ pick_port.stdout | trim }}" - _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - - - name: Build tunnel control socket path - delegate_to: localhost - ansible.builtin.set_fact: - _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - - - name: Debug chosen tunnel params - delegate_to: localhost - ansible.builtin.debug: - msg: - - "local_port={{ _local_port }}" - - "ctrl_dir={{ _ctrl_dir }}" - - "ctrl_sock={{ _ctrl_sock }}" - - - name: issue arping with own IP - ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 - - # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - - name: Start SSH tunnel via DEV1 background ControlMaster - delegate_to: localhost - ansible.builtin.shell: | - set -e - USER="{{ dev1_user }}" - HOST="{{ ansible_host | default(inventory_hostname) }}" - sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ - -M -S "{{ _ctrl_sock }}" \ - -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ - "${USER}@${HOST}" - args: - executable: /bin/bash - register: start_tunnel - changed_when: true - - - name: Small delay for tunnel to settle - delegate_to: localhost - ansible.builtin.wait_for: - timeout: 1 - changed_when: false - - - name: Verify tunnel master running proper O check with destination - delegate_to: localhost - ansible.builtin.shell: | - set -e - HOST="{{ ansible_host | default(inventory_hostname) }}" - ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true - register: tun_check - changed_when: false - - - name: Debug tunnel check - delegate_to: localhost - ansible.builtin.debug: - msg: - - "tunnel_check.rc={{ tun_check.rc }}" - - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" - - # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - - name: Sanity confirm tunnel TCP reachability to DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - nc -z -w5 127.0.0.1 "{{ _local_port }}" - register: nc_probe - changed_when: false - ignore_errors: true - - - name: Debug reachability result - delegate_to: localhost - ansible.builtin.debug: - msg: - - "nc.rc={{ nc_probe.rc }}" - - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - - - name: Stop if tunnel TCP check failed - ansible.builtin.meta: end_host - when: nc_probe.rc != 0 - - - name: Sanity show presence and permissions of passfiles on controller - delegate_to: localhost - ansible.builtin.shell: | - set -e - ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" - register: dev2_ls - changed_when: false - - - name: Debug passfiles listing - delegate_to: localhost - ansible.builtin.debug: - msg: - - "{{ (dev2_ls.stdout | default('')) | trim }}" - - "{{ (dev2_ls.stderr | default('')) | trim }}" - - # ---------------------------- Pick DEV2 password root only ---------------------------- - - name: Try DEV2 login with basicpass root - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass - changed_when: false - ignore_errors: true - - - name: Select basicpass as working passfile - when: dev2_try_basicpass.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass" - changed_when: false - - - name: Try DEV2 login with basicpass2 root only if first failed - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f basicpass2 ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 - register: dev2_try_basicpass2 - changed_when: false - ignore_errors: true - - - name: Select basicpass2 as working passfile - when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "basicpass2" - changed_when: false - - - name: Set dev2_passfile_used to NONE if neither worked - when: dev2_passfile_used is not defined - delegate_to: localhost - ansible.builtin.set_fact: - dev2_passfile_used: "NONE" - changed_when: false - - - name: Debug selected DEV2 passfile or NONE - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - - - name: Read DEV2 hostname via tunnel (busybox-safe) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" - args: - executable: /bin/bash - register: dev2_host_read - changed_when: false - - - name: Normalize hostnames for strict compare - ansible.builtin.set_fact: - _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" - _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - - - name: Debug hostname bytes (hex) - delegate_to: localhost - ansible.builtin.debug: - msg: - - "inv={{ _inv_hn | tojson }}" - - "dev1={{ _dev1_hn | tojson }}" - - "dev2={{ _dev2_hn | tojson }}" - - # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - - name: Enforce DEV2 hostname equals inventory and DEV1 - block: - - name: Fail if DEV2 hostname differs from inventory/DEV1 - ansible.builtin.fail: - msg: > - Hostname mismatch: DEV2='{{ _dev2_hn }}', - inventory='{{ _inv_hn }}', - DEV1='{{ _dev1_hn }}' - when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) - rescue: - - name: Initialize journal array for hostname mismatch - ansible.builtin.set_fact: - _journal: [] - _blocked: true - _prep_blocked: false - delegate_to: localhost - - - name: Append hostname mismatch to journal - ansible.builtin.set_fact: - _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" - delegate_to: localhost - - - name: Build control queue payload for indoor aborted journal (hostname) - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue (hostname) - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_hn_resp - changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_hn_resp.status != 200) or - (rmq_journal_indoor_aborted_hn_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) - delegate_to: localhost - - - name: Stop host after hostname mismatch - ansible.builtin.meta: end_host - - # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - - name: Initialize journal and flags - ansible.builtin.set_fact: - _journal: [] - _prep_blocked: false - _blocked: false - - - name: Build specific marker filename (no .bin) - ansible.builtin.set_fact: - _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - - - name: Check for any preparation markers on DEV2 (wildcard count) - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "ls /tmp/prepared_for* 2>/dev/null | wc -l" - args: - executable: /bin/bash - register: dev2_prep_count - changed_when: false - ignore_errors: true - - - name: Soft-block if any prep markers already present - when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" - - # ---------------------------- DEV2 version firmux primary check ---------------------------- - - name: Read DEV2 firmux release - when: dev2_passfile_used != "NONE" - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "cat /usr/lib/release/firmux 2>/dev/null || true" - args: - executable: /bin/bash - register: dev2_firmux - changed_when: false - ignore_errors: true - - - name: Debug firmux content DEV2 - when: dev2_firmux is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - # ---------------------------- Normalize rebootin early ---------------------------- - - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" - _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" - _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" - _reboot_minutes: >- - {{ - 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) - else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) - }} - - - name: "Normalize rebootin (phase 2: requested flag)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" - - # >>> NEW: compute effective seconds (0 min → 20 seconds) - - name: "Normalize rebootin (phase 3: compute seconds)" - delegate_to: localhost - ansible.builtin.set_fact: - _reboot_seconds: >- - {{ - 20 if (_reboot_requested | default(false)) and ((_reboot_minutes | int) == 0) - else ((_reboot_minutes | int) * 60) - }} - - # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - - name: Build control queue payload for indoor start journal - ansible.builtin.set_fact: - journal_indoor_start: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 is reachable, starting with update. - image={{ image_filename }}, md5={{ image_md5 }}, - firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, - reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} - when: dev2_passfile_used != "NONE" - delegate_to: localhost - - - name: Publish indoor start journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_start | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_start_resp - changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_start_resp.status != 200) or - (rmq_journal_indoor_start_resp.json is not defined) or - (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) - when: journal_indoor_start is defined - delegate_to: localhost - - # ---------------------------- Stage image on DEV2 no write ---------------------------- - - name: Check local presence of image file - delegate_to: localhost - ansible.builtin.stat: - path: "{{ image_filename }}" - register: local_img - - - name: Soft-block if image missing locally - when: not local_img.stat.exists - ansible.builtin.set_fact: - _blocked: true - _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - - - name: Compute local md5 of the image - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.shell: | - set -e - md5sum "{{ image_filename }}" | awk '{print $1}' - register: local_md5 - changed_when: false - - - name: Verify local md5 matches expected - when: local_img.stat.exists - delegate_to: localhost - ansible.builtin.assert: - that: - - (local_md5.stdout | trim) == image_md5 - fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" - success_msg: "Local md5 matches expected" - - # Optional SHA256 local - - name: Compute local sha256 of the image (if provided) - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - sha256sum "{{ image_filename }}" | awk '{print $1}' - register: local_sha256 - changed_when: false - ignore_errors: true - - - name: Soft-block if local sha256 mismatch or unavailable - when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - # fw_printenv health before upload (soft-fail) - - name: Read fw_printenv size (line count) on DEV2 - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "(fw_printenv 2>/dev/null | wc -l) || echo 0" - args: - executable: /bin/bash - register: dev2_fwenv_wc - changed_when: false - ignore_errors: true - - - name: Soft-block if fw_printenv too small - when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - - - name: Check existing image md5 on DEV2 NOFILE if absent - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_before - changed_when: false - - - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" scp \ - -P "$PORT" \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" - args: - executable: /bin/bash - register: scp_push - changed_when: true - - - name: Compute md5 of image on DEV2 after copy or if exists - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" - args: - executable: /bin/bash - register: dev2_md5_after - changed_when: false - - - name: Verify DEV2 md5 matches expected (soft-fail journal) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" - - # Optional SHA256 remote - - name: Compute sha256 of image on DEV2 (if provided) - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" - args: - executable: /bin/bash - register: dev2_sha256_after - changed_when: false - ignore_errors: true - - - name: Soft-block if DEV2 sha256 mismatch/unavailable - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - - - name: Run noninvasive update check on DEV2 update c - when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=15 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -c '{{ dev2_image_path }}' 2>&1 || true" - args: - executable: /bin/bash - register: dev2_update_check - changed_when: false - - - name: Debug update c output DEV2 - when: dev2_update_check is defined - delegate_to: localhost - ansible.builtin.debug: - msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - - - name: Soft-block if update -c did not say valid - when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) - ansible.builtin.set_fact: - _prep_blocked: true - _blocked: true - _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" - - # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - - name: Build control queue payload for indoor aborted journal - ansible.builtin.set_fact: - journal_indoor_aborted: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} - when: (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor aborted journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_aborted | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_aborted_resp - changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_aborted_resp.status != 200) or - (rmq_journal_indoor_aborted_resp.json is not defined) or - (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) - when: journal_indoor_aborted is defined - delegate_to: localhost - - # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - - name: Upgrade write and flip block - when: not (_blocked | default(false)) - block: - - name: Write image on DEV2 (this will take a while) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=0 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "update -w '{{ dev2_image_path }}'" - args: - executable: /bin/bash - register: dev2_up_write - changed_when: true - failed_when: dev2_up_write.stdout is not search('update is complete') - - - name: Read current active partition on DEV2 - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_before - changed_when: false - failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - - - name: Determine new active value for DEV2 - when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) - ansible.builtin.set_fact: - dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - - - name: Switch active partition to {{ dev2_new_active }} - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_setenv active {{ dev2_new_active }}" - args: - executable: /bin/bash - register: dev2_setenv_out - changed_when: true - - - name: Verify active partition flipped on DEV2 - when: dev2_new_active is defined - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "fw_printenv active | awk -F= '/^active=/{print \$2}'" - args: - executable: /bin/bash - register: dev2_active_after - changed_when: false - failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - - - name: Create specific prep marker on DEV2 (no generic) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ - "touch '{{ _marker_specific }}'" - args: - executable: /bin/bash - register: dev2_marker_write - changed_when: true - ignore_errors: true - - # ---------------------------- Reboot scheduling (normalized) ---------------------------- - - name: Schedule delayed reboot on DEV2 minutes - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - ansible.builtin.shell: | - set -e - PORT="{{ _local_port }}" - SECS="{{ _reboot_seconds | int }}" - CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' - sshpass -f "{{ dev2_passfile_used }}" ssh \ - -o AddressFamily=inet \ - -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ - -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ - -o ConnectTimeout=10 \ - -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" - args: - executable: /bin/bash - register: dev2_reboot_sched - changed_when: true - ignore_errors: true - - - name: Build control queue payload for indoor-restart-scheduled tag - ansible.builtin.set_fact: - tag_restart_sched_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "tag_add" - task_result: "indoor-restart-scheduled" - when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) - delegate_to: localhost - - - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ tag_restart_sched_payload | to_json }}" - payload_encoding: "string" - register: rmq_tag_restart_sched_resp - changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_tag_restart_sched_resp.status != 200) or - (rmq_tag_restart_sched_resp.json is not defined) or - (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) - when: tag_restart_sched_payload is defined - delegate_to: localhost - - - - name: Note reboot request bad format - when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested - ansible.builtin.debug: - msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" - - # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - - name: Build control queue payload for indoor updated journal - ansible.builtin.set_fact: - _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" - when: not (_blocked | default(false)) - delegate_to: localhost - - - name: Build indoor updated journal payload text - ansible.builtin.set_fact: - journal_indoor_updated: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - Indoor: Dev2 updated, scheduling reboot {{ - ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) - else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) - else 'not requested' - }}. - write_done={{ _write_success }}, - active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, - active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Publish indoor updated journal to control queue - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ journal_indoor_updated | to_json }}" - payload_encoding: "string" - register: rmq_journal_indoor_updated_resp - changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_journal_indoor_updated_resp.status != 200) or - (rmq_journal_indoor_updated_resp.json is not defined) or - (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) - when: journal_indoor_updated is defined - delegate_to: localhost - - - - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) - ansible.builtin.set_fact: - au_attempt: 1 - au_max_attempts: 3 - au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" - when: not (_blocked | default(false)) and (_write_success | bool) - delegate_to: localhost - - - name: Generate correlation ID and original emitted timestamp - ansible.builtin.set_fact: - au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" - au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" - when: au_delay_sec is defined - delegate_to: localhost - - - name: Build after-upgrade check payload (attempt 1) - ansible.builtin.set_fact: - afterupgrade_payload: - task_name: "afterupgrade_indoor_check" - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" - attempt: "{{ au_attempt }}" - max_attempts: "{{ au_max_attempts }}" - current_delay_sec: "{{ au_delay_sec }}" - correlation_id: "{{ au_correlation_id }}" - original_emitted_at: "{{ au_original_emitted_at }}" - schema_version: 1 - when: au_delay_sec is defined - delegate_to: localhost - - - name: Debug x-delay about to be sent (ms) - ansible.builtin.debug: - msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" - when: afterupgrade_payload is defined - delegate_to: localhost - - - name: Publish delayed after-upgrade check to holding exchange - ansible.builtin.uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - headers: - x-delay: "{{ (au_delay_sec | int) * 1000 }}" - routing_key: "{{ afterupgrade_routing_key }}" - payload: "{{ afterupgrade_payload | to_json }}" - payload_encoding: "string" - register: rmq_afterupgrade_resp - changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) - failed_when: > - (rmq_afterupgrade_resp.status != 200) or - (rmq_afterupgrade_resp.json is not defined) - when: afterupgrade_payload is defined - delegate_to: localhost - - - # ---------------------------- summary ---------------------------- - - name: Summary show key results - delegate_to: localhost - ansible.builtin.debug: - msg: - - "dev2_passfile_used={{ dev2_passfile_used }}" - - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - - "local_image_present={{ local_img.stat.exists | default(false) }}" - - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - - "reboot_requested={{ _reboot_requested | default(false) }}" - - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" - - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - - "prep_blocked={{ _prep_blocked | default(false) }}" - - "blocked={{ _blocked | default(false) }}" - - "journal={{ (_journal | default([])) | join(' || ') }}" - - post_tasks: - - name: Cleanup always - block: - - ansible.builtin.debug: - msg: "entering cleanup block" - changed_when: false - delegate_to: localhost - always: - - name: Close tunnel best effort - delegate_to: localhost - ansible.builtin.shell: | - ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true - changed_when: false - ignore_errors: true - - - name: Remove control dir best effort - delegate_to: localhost - ansible.builtin.file: - path: "{{ _ctrl_dir | default('/tmp/none') }}" - state: absent - ignore_errors: true - - - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK - ansible.builtin.raw: > - {{ pathprefix }} - ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} - register: del_ip - changed_when: del_ip.rc == 0 - failed_when: > - del_ip.rc != 0 - and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) - and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - - - name: Debug del ip results - ansible.builtin.debug: - msg: - - "del_ip.rc={{ del_ip.rc | default('') }}" - - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" - when: del_ip is defined diff --git a/files/ansible-playbooks/wifidebug17.yml-old b/files/ansible-playbooks/wifidebug17.yml-old deleted file mode 100644 index 4184be4..0000000 --- a/files/ansible-playbooks/wifidebug17.yml-old +++ /dev/null @@ -1,441 +0,0 @@ ---- -- name: Deploy WiFi debug (single device, linear) - hosts: all - gather_facts: no - - vars: - remote_syslog_ip: "102.38.125.161" - - # Program version (0) - wdbg_version: "v17" - - ssh_user: "{{ ansible_user | default('root') }}" - ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" - - # RabbitMQ (use controls exchange + queue_controls like the reference) (e) - rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" - rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" - rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" - rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" - rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" - rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" - control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" - - tasks: - - - block: - - # --- SSH reachability check --- - - name: Check SSH connectivity (raw ping) - raw: "echo ping" - register: ping_result - ignore_errors: true - - - block: - -############ step 1 - - - name: Check if crontab launch is already present in sysstart.lua - raw: | - grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT - register: sysstart_cron_check - changed_when: false - failed_when: false - - - name: Announce presence and skip edits - when: (sysstart_cron_check.stdout | trim) == 'PRESENT' - debug: - msg: "crontab is present in sysstart.lua; skipping download/edit/upload." - - - name: Fetch sysstart.lua via scp (password auth) - when: (sysstart_cron_check.stdout | trim) != 'PRESENT' - delegate_to: localhost - command: > - sshpass -p {{ ssh_pass | quote }} - scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua - /tmp/{{ inventory_hostname }}_sysstart.lua - register: scp_get - retries: 3 - delay: 2 - until: scp_get.rc == 0 - - - name: Edit the file locally - when: (sysstart_cron_check.stdout | trim) != 'PRESENT' - delegate_to: localhost - lineinfile: - path: "/tmp/{{ inventory_hostname }}_sysstart.lua" - line: 'launchd.set_process("crond", "/usr/sbin/crond -f")' - insertbefore: '^tasks\.start\(ctx,\s*"finished"\)' - - - name: Push the file back (write to -2) - when: (sysstart_cron_check.stdout | trim) != 'PRESENT' - delegate_to: localhost - command: > - sshpass -p {{ ssh_pass | quote }} - scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - /tmp/{{ inventory_hostname }}_sysstart.lua - {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua - register: scp_put - retries: 3 - delay: 2 - until: scp_put.rc == 0 - -############ step 2 - - - name: Compute MD5 of local wifidebug.sh - delegate_to: localhost - command: md5sum files/wifidebug.sh - register: md5_local_wdbg - changed_when: false - - - name: Compute MD5 of remote /root/wifidebug.sh - raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh" - register: md5_remote_wdbg - changed_when: false - failed_when: false - - - name: Decide if wifidebug.sh needs upload - set_fact: - upload_wdbg: >- - {{ (md5_remote_wdbg.rc != 0) - or ((md5_local_wdbg.stdout.split()[0]) - != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }} - - - name: Upload wifidebug.sh via scp (overwrite if changed) - when: upload_wdbg | bool - delegate_to: localhost - command: > - sshpass -p {{ ssh_pass | quote }} - scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - files/wifidebug.sh - {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh - register: scp_wifidebug - retries: 3 - delay: 2 - until: scp_wifidebug.rc == 0 - - - name: Ensure /root/wifidebug.sh is executable and owned by root - raw: | - chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh - -############ step 3 - - - name: Ensure /etc/crontabs/root exists (touch with perms) - raw: | - if [ ! -f /etc/crontabs/root ]; then - touch /etc/crontabs/root - fi - chown root:root /etc/crontabs/root - chmod 0644 /etc/crontabs/root - - - name: Check if cron line already present - raw: | - grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root - register: cron_grep - failed_when: false - changed_when: false - - - name: Upload snippet crond-root to /tmp (only if missing) - when: cron_grep.rc != 0 - delegate_to: localhost - command: > - sshpass -p {{ ssh_pass | quote }} - scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null - files/crond-root - {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet - - - name: Append snippet to /etc/crontabs/root (only if missing) - when: cron_grep.rc != 0 - raw: | - cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet - register: cron_append - changed_when: true - -############ step 4 - - - name: Skip config.json edit if remote_syslog already matches - raw: | - EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0) - SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0) - if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then - echo "SKIP" - else - echo "EDIT" - fi - register: remote_syslog_check - changed_when: false - - - name: Set do_edit flag from skip check - set_fact: - do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}" - - - name: Fetch /tmp/config.json to controller - when: do_edit - delegate_to: localhost - command: > - sshpass -p {{ ssh_pass | quote }} - scp -q -o StrictHostKeyChecking=no - {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json - /tmp/{{ inventory_hostname }}_config.json - - - name: Strict in-place style-preserving edit (services.remote_syslog only) - when: do_edit - delegate_to: localhost - shell: | - awk ' - BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 } - { - line = $0 - if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 } - if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 } - if (in_rs) { - if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) { - line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line) - done_enabled=1 - } - if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) { - line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line) - done_server=1 - } - } - print line - if (in_rs) { - opens = gsub(/{/, "{", $0) - closes = gsub(/}/, "}", $0) - depth += (opens - closes) - if (depth <= 0 && $0 ~ /}/) { in_rs=0 } - } - } - ' "/tmp/{{ inventory_hostname }}_config.json" \ - > "/tmp/{{ inventory_hostname }}_config.json.new" - args: - executable: /bin/bash - - - name: Push edited config.json.new to remote temp - when: do_edit - delegate_to: localhost - shell: | - sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \ - "/tmp/{{ inventory_hostname }}_config.json.new" \ - {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new - args: - executable: /bin/bash - - - name: Compute MD5 of local .new - when: do_edit - delegate_to: localhost - command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new" - register: md5_local - changed_when: false - - - name: Compute MD5 of remote .new (no Python) - when: do_edit - raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new" - register: md5_remote - changed_when: false - - - name: Fail if MD5 mismatch - when: - - do_edit - - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD') - fail: - msg: "MD5 mismatch between controller and remote copy — aborting replace." - - - name: Commit new config.json (overwrite original) - when: do_edit - raw: | - cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S) - mv /tmp/config.json.new /tmp/config.json - - - name: Apply config to runtime (sysconf -w) - when: do_edit - raw: | - sysconf -w || /usr/sbin/sysconf -w - register: sysconf_result - changed_when: true - failed_when: sysconf_result.rc not in [0] - - # (d) Pre-restart notice to control queue - - name: Build pre-restart journal (explain why) - when: do_edit - delegate_to: localhost - set_fact: - prereboot_payload: - inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" - task_name: "journal_add" - task_result: >- - wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w). - - - name: Publish pre-restart journal to control queue - when: do_edit - delegate_to: localhost - uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ prereboot_payload | to_json }}" - payload_encoding: "string" - register: rmq_preboot - changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool) - - # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ======== - - - name: Trigger full restart (system-stop; system-start) from controller with 10s cap - when: do_edit - delegate_to: localhost - shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'" - args: - executable: /bin/bash - register: restart_kick - changed_when: true - failed_when: false - - - # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3) - - name: Probe SSH with nc (24 tries, 5s each) - when: do_edit - delegate_to: localhost - shell: "nc -z -w 3 {{ ansible_host }} 22" - register: nc_probe - retries: 24 - delay: 5 - until: nc_probe.rc == 0 - changed_when: false - failed_when: false - - - name: Set ssh_up fact (from nc probe) - when: do_edit - delegate_to: localhost - set_fact: - ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}" - - - name: Fail if SSH did not return after restart - when: - - do_edit - - not ssh_up | bool - fail: - msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks." - - # ======== END OF SURGICAL CHANGE ======== - - - name: Set result status (success deployed or no change) - set_fact: - result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}" - - when: ping_result is succeeded - - - name: Set status fact (no ssh) - when: ping_result is failed - set_fact: - result_status: "NO_SSH" - - rescue: - - name: Mark result as failed - set_fact: - result_status: "FAILED during {{ ansible_failed_task.name }}" - - always: - - - name: Compute inscope device - set_fact: - inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}" - - # (a) Set custom field wifidebug -> v15 on control queue - - - name: Build custom-field payload (wifidebug -> version) - delegate_to: localhost - set_fact: - wdbg_cf_payload: - inscope_device: "{{ inscope_device_name }}" - task_name: "deploy_wifidebug" - task_result: "{{ wdbg_version }}" - - - name: Publish custom-field update to control queue - delegate_to: localhost - uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ wdbg_cf_payload | to_json }}" - payload_encoding: "string" - register: rmq_cf - changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool) - - # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed - - - name: Build actions list - set_fact: - _actions_list: >- - {{ - [] - + ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], [])) - + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], [])) - + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], [])) - + ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], [])) - + ((do_edit | default(false) | bool) | ternary(['restarted services'], [])) - }} - - - name: Build actions string - set_fact: - _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}" - - - - name: Build wrap-up journal payload - delegate_to: localhost - set_fact: - wrap_payload: - inscope_device: "{{ inscope_device_name }}" - task_name: "journal_add" - task_result: >- - wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }} - — actions: {{ _actions_str }} - - - name: Publish wrap-up journal to control queue - delegate_to: localhost - uri: - url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" - method: POST - user: "{{ rmq_user }}" - password: "{{ rmq_pass }}" - force_basic_auth: true - status_code: 200 - headers: - content-type: "application/json" - body_format: json - body: - properties: - content_type: "application/json" - routing_key: "{{ control_queue }}" - payload: "{{ wrap_payload | to_json }}" - payload_encoding: "string" - register: rmq_wrap - changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool) - - # Local summary (kept for operator visibility) - - name: Summary - debug: - msg: - - "result_status: {{ result_status }}" - - "we're good" -