diff --git a/files/ansible-playbooks/afterupgrade_withcurl.yml b/files/ansible-playbooks/afterupgrade_withcurl.yml new file mode 100644 index 0000000..885c8c6 --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_withcurl.yml @@ -0,0 +1,377 @@ +--- +- name: After-upgrade verification (banner check + reporting) + hosts: all + gather_facts: no + + # RabbitMQ + defaults (match the big script) + vars: + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + rmq_exchange: "controls" + control_queue: "queue_controls" + + # --- Manual run defaults (so we can execute without -e) --- + # These are safe to leave here; anything passed via -e will still override them. + attempt: 1 + max_attempts: 3 + current_delay_sec: 300 + correlation_id: "6f680073dc7c" + original_emitted_at: "2025-10-30T18:46:52Z" + target_version: "2.2.3 rev 9800" + # Intentionally keep this empty to exercise the target_version_effective logic. + target_version_full: "" + schema_version: 1 + + + + # Probing/SSH defaults + tcp_port: 22 + nc_timeout: 5 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + ssh_timeout: 10 + + # Do NOT self-reference max_attempts. We’ll normalize below. + max_attempts_default: 3 + + tasks: + # ---- Normalize metadata safely (no self-referential defaults) ---- + - name: Normalize metadata (no clever transforms) + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + # preserve the original string verbatim for all subsequent retries + target_version_full: "{{ target_version | default('') }}" + + - name: Derive effective target version (avoid extra-var masking) + ansible.builtin.set_fact: + target_version_effective: >- + {{ (target_version_full | default('') | trim) + if (target_version_full | default('') | trim) + else (target_version | default('') | trim) }} + + - name: Show received metadata + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "max_attempts={{ effective_max_attempts }}" + - "correlation_id={{ correlation_id }}" + - "original_emitted_at={{ original_emitted_at }}" + - "target_version(full)={{ target_version_full }}" + + # ---- Fast TCP reachability probe (controller-side) ---- + - name: Check if TCP/{{ tcp_port }} is reachable with nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Build failure journal (no TCP connectivity) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed). + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (no TCP connectivity) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). + - name: Stop host after TCP failure (we’ll schedule or close out below) + when: nc_probe.rc != 0 + ansible.builtin.meta: noop + + # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- + - name: Probe banner via SSH from controller (classic extraction) + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ ssh_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ ssh_pass }}' \ + ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ + "${USER}@${HOST}" \ + "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" + register: banner_probe + changed_when: false + ignore_errors: true + + - name: Show the current version (banner line) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.debug: + msg: "{{ banner_probe.stdout | trim }}" + + - name: Evaluate version match (full-string contains check) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.set_fact: + version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}" + + + # ---- Journaling paths ---- + # Success: banner matches expected full target_version + - name: Build success journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_success + changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) + + # NEW: send a control tag to clean up device state on success + - name: Build cleanup control payload (update_cleanup_success) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + control_cleanup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "update_cleanup_success" + + - name: Publish cleanup control message to control queue + when: control_cleanup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ control_cleanup_payload | to_json }}" + payload_encoding: "string" + register: rmq_cleanup_success + changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) + + # Mismatch: reachable & banner read, but not equal to target_version + - name: Build mismatch journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_mismatch_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish mismatch journal to control queue + when: journal_mismatch_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_mismatch_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_mismatch + changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) + + # SSH error path: TCP OK, but SSH failed + - name: Build failure journal payload (ssh error) + mark retry + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): + {{ (banner_probe.stderr | default('') | trim) }} + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (ssh error) to control queue + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_fail + changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) + + # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- + - name: Compute next-attempt delay (ms) according to policy + when: (_needs_retry | default(false)) | bool + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: >- + {% if attempt | int == 1 %} + 300 + {% elif attempt | int == 2 %} + 600 + {% else %} + 0 + {% endif %} + next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" + + # If we've reached the cap, send a final “gave up” journal and stop. + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + # Otherwise schedule the next attempt (only if we still have budget) + - name: Build delayed after-upgrade payload for next attempt + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host