438 lines
19 KiB
YAML
438 lines
19 KiB
YAML
---
|
||
- name: After-upgrade verification (banner check + reporting)
|
||
hosts: all
|
||
gather_facts: no
|
||
|
||
# RabbitMQ + defaults (match the big script)
|
||
vars:
|
||
rmq_host: "10.210.12.2"
|
||
rmq_port: 15672
|
||
rmq_user: "admin"
|
||
rmq_pass: "change_me"
|
||
rmq_vhost: "app"
|
||
rmq_exchange: "controls"
|
||
control_queue: "queue_controls"
|
||
|
||
# --- Manual run defaults (so we can execute without -e) ---
|
||
# These are safe to leave here; anything passed via -e will still override them.
|
||
attempt: 1
|
||
max_attempts: 3
|
||
current_delay_sec: 300
|
||
correlation_id: "6f680073dc7c"
|
||
original_emitted_at: "2025-10-30T18:46:52Z"
|
||
target_version: "2.2.3 rev 9800"
|
||
# Intentionally keep this empty to exercise the target_version_effective logic.
|
||
target_version_full: ""
|
||
schema_version: 1
|
||
|
||
|
||
# Probing/SSH defaults
|
||
tcp_port: 22
|
||
nc_timeout: 5
|
||
ssh_user: "{{ ansible_user | default('root') }}"
|
||
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
|
||
ssh_timeout: 10
|
||
|
||
# Do NOT self-reference max_attempts. We’ll normalize below.
|
||
max_attempts_default: 3
|
||
|
||
# --- Hardcoded cloud API bearer (per request) ---
|
||
cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg5MTk2NTQ2LCJleHAiOjE3OTE3ODg1NDZ9.fQlcMWTL4uTKewd-AjlaHOdXSMJ2JOw2RfnKKF3weWk"
|
||
|
||
tasks:
|
||
# ---- Normalize metadata safely (no self-referential defaults) ----
|
||
- name: Normalize metadata (no clever transforms)
|
||
ansible.builtin.set_fact:
|
||
attempt: "{{ (attempt | default(1)) | int }}"
|
||
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
|
||
correlation_id: "{{ correlation_id | default('') }}"
|
||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
||
target_version: "{{ target_version | default('') }}"
|
||
# preserve the original string verbatim for all subsequent retries
|
||
target_version_full: "{{ target_version | default('') }}"
|
||
|
||
- name: Derive effective target version (avoid extra-var masking)
|
||
ansible.builtin.set_fact:
|
||
target_version_effective: >-
|
||
{{ (target_version_full | default('') | trim)
|
||
if (target_version_full | default('') | trim)
|
||
else (target_version | default('') | trim) }}
|
||
|
||
- name: Show received metadata
|
||
ansible.builtin.debug:
|
||
msg:
|
||
- "attempt={{ attempt }}"
|
||
- "max_attempts={{ effective_max_attempts }}"
|
||
- "correlation_id={{ correlation_id }}"
|
||
- "original_emitted_at={{ original_emitted_at }}"
|
||
- "target_version(full)={{ target_version_full }}"
|
||
|
||
# ---- Fast TCP reachability probe (controller-side) ----
|
||
- name: Check if TCP/{{ tcp_port }} is reachable with nc
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
|
||
register: nc_probe
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Build failure journal (no TCP connectivity) + mark retry
|
||
when: nc_probe.rc != 0
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
fail_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
|
||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
||
_needs_retry: true
|
||
|
||
- name: Publish failure journal (no TCP connectivity)
|
||
when: nc_probe.rc != 0
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ fail_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_j_tcp_fail
|
||
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
|
||
|
||
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
|
||
- name: Stop host after TCP failure (we’ll schedule or close out below)
|
||
when: nc_probe.rc != 0
|
||
ansible.builtin.meta: noop
|
||
|
||
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
|
||
- name: Probe banner via SSH from controller (classic extraction)
|
||
when: nc_probe.rc == 0
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
USER="{{ ssh_user }}"
|
||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
||
sshpass -p '{{ ssh_pass }}' \
|
||
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
|
||
"${USER}@${HOST}" \
|
||
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
|
||
register: banner_probe
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Show the current version (banner line)
|
||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
||
ansible.builtin.debug:
|
||
msg: "{{ banner_probe.stdout | trim }}"
|
||
|
||
- name: Evaluate version match (full-string contains check)
|
||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
||
ansible.builtin.set_fact:
|
||
version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"
|
||
|
||
# ---- Read eth0 MAC (only after confirmed version match) ----
|
||
- name: Read eth0 MAC address via SSH
|
||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
USER="{{ ssh_user }}"
|
||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
||
sshpass -p '{{ ssh_pass }}' \
|
||
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
|
||
"${USER}@${HOST}" \
|
||
"cat /sys/class/net/eth0/address | tr -d '\n'"
|
||
register: mac_probe
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Set eth0_macaddress fact
|
||
when: mac_probe is defined and mac_probe.rc == 0
|
||
ansible.builtin.set_fact:
|
||
eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"
|
||
|
||
- name: Debug eth0_macaddress
|
||
when: eth0_macaddress is defined
|
||
ansible.builtin.debug:
|
||
msg: "eth0_macaddress={{ eth0_macaddress }}"
|
||
|
||
# ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
|
||
- name: Build URL-encoded MAC for cloud API
|
||
when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
|
||
ansible.builtin.set_fact:
|
||
enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"
|
||
|
||
- name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
|
||
when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
|
||
delegate_to: localhost
|
||
ansible.builtin.shell: |
|
||
set -e
|
||
curl -sS -L --request PATCH --post301 --post302 \
|
||
"https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
|
||
--header "Authorization: Bearer {{ cloud_api_bearer }}" \
|
||
--header "Content-Type: application/json" \
|
||
--header "Accept: application/json" \
|
||
--fail-with-body \
|
||
--data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
|
||
register: cloud_patch
|
||
changed_when: false
|
||
ignore_errors: true
|
||
|
||
- name: Flag cloud change result
|
||
when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
|
||
ansible.builtin.set_fact:
|
||
cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"
|
||
|
||
- name: Debug cloud change result
|
||
when: cloud_change_ok is defined
|
||
ansible.builtin.debug:
|
||
msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"
|
||
|
||
# ---- Journaling paths ----
|
||
# Success: banner matches expected full target_version
|
||
- name: Build success journal payload
|
||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
journal_success_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
||
Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
|
||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
||
{{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}
|
||
|
||
- name: Publish success journal to control queue
|
||
when: journal_success_payload is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_success_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_j_success
|
||
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
|
||
|
||
# NEW: send a control tag to clean up device state on success
|
||
- name: Build cleanup control payload (update_cleanup_success)
|
||
when: journal_success_payload is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
control_cleanup_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "update_cleanup_success"
|
||
|
||
- name: Publish cleanup control message to control queue
|
||
when: control_cleanup_payload is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ control_cleanup_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_cleanup_success
|
||
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
|
||
|
||
# Mismatch: reachable & banner read, but not equal to target_version
|
||
- name: Build mismatch journal payload
|
||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
journal_mismatch_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
||
Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
|
||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
||
|
||
- name: Publish mismatch journal to control queue
|
||
when: journal_mismatch_payload is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_mismatch_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_j_mismatch
|
||
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
|
||
|
||
# SSH error path: TCP OK, but SSH failed
|
||
- name: Build failure journal payload (ssh error) + mark retry
|
||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
journal_fail_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
||
{{ (banner_probe.stderr | default('') | trim) }}
|
||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
||
_needs_retry: true
|
||
|
||
- name: Publish failure journal (ssh error) to control queue
|
||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_fail_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_j_fail
|
||
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
|
||
|
||
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
|
||
- name: Compute next-attempt delay (ms) according to policy
|
||
when: (_needs_retry | default(false)) | bool
|
||
ansible.builtin.set_fact:
|
||
next_attempt: "{{ attempt | int + 1 }}"
|
||
next_delay_sec: >-
|
||
{% if attempt | int == 1 %}
|
||
300
|
||
{% elif attempt | int == 2 %}
|
||
600
|
||
{% else %}
|
||
0
|
||
{% endif %}
|
||
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
|
||
|
||
# If we've reached the cap, send a final “gave up” journal and stop.
|
||
- name: Build final gave-up journal (max attempts reached)
|
||
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
journal_gaveup_payload:
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
task_name: "journal_add"
|
||
task_result: >-
|
||
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
||
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
|
||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
||
|
||
- name: Publish final gave-up journal
|
||
when: journal_gaveup_payload is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
routing_key: "{{ control_queue }}"
|
||
payload: "{{ journal_gaveup_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_j_gaveup
|
||
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
||
|
||
- name: Stop host after final gave-up
|
||
when: journal_gaveup_payload is defined
|
||
ansible.builtin.meta: end_host
|
||
|
||
# Otherwise schedule the next attempt (only if we still have budget)
|
||
- name: Build delayed after-upgrade payload for next attempt
|
||
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
|
||
delegate_to: localhost
|
||
ansible.builtin.set_fact:
|
||
delayed_payload:
|
||
task_name: "afterupgrade_check"
|
||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
||
attempt: "{{ next_attempt | int }}"
|
||
max_attempts: "{{ effective_max_attempts | int }}"
|
||
correlation_id: "{{ correlation_id }}"
|
||
original_emitted_at: "{{ original_emitted_at }}"
|
||
target_version: "{{ target_version_full }}"
|
||
current_delay_sec: "{{ next_delay_sec | int }}"
|
||
schema_version: 1
|
||
|
||
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
|
||
when: delayed_payload is defined
|
||
delegate_to: localhost
|
||
ansible.builtin.uri:
|
||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
||
method: POST
|
||
user: "{{ rmq_user }}"
|
||
password: "{{ rmq_pass }}"
|
||
force_basic_auth: true
|
||
status_code: 200
|
||
headers:
|
||
content-type: "application/json"
|
||
body_format: json
|
||
body:
|
||
properties:
|
||
content_type: "application/json"
|
||
expiration: "{{ (next_delay_ms | int) | string }}"
|
||
correlation_id: "{{ correlation_id }}"
|
||
routing_key: "deviceconfig"
|
||
payload: "{{ delayed_payload | to_json }}"
|
||
payload_encoding: "string"
|
||
register: rmq_pub_next
|
||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
||
|
||
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
|
||
when: (_needs_retry | default(false)) | bool
|
||
ansible.builtin.meta: end_host
|