pavel/ansible-worker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
ansible user
2025-10-22 14:11:49 +02:00
commit 110301f862
75 changed files with 20802 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
files/ansible-playbooks/afterupgrade_check.yml-from_chat Normal file
View File

@@ -0,0 +1,355 @@
---
- name: After-upgrade verification (banner check + reporting)
hosts: all
gather_facts: no
# RabbitMQ + defaults (match the big script)
vars:
rmq_host: "10.210.12.2"
rmq_port: 15672
rmq_user: "admin"
rmq_pass: "change_me"
rmq_vhost: "app"
rmq_exchange: "controls"
control_queue: "queue_controls"
# Probing/SSH defaults
tcp_port: 22
nc_timeout: 5
ssh_user: "{{ ansible_user | default('root') }}"
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
ssh_timeout: 10
# Do NOT self-reference max_attempts. We’ll normalize below.
max_attempts_default: 3
tasks:
# ---- Normalize metadata safely (no self-referential defaults) ----
- name: Normalize metadata (no clever transforms)
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
target_version: "{{ target_version | default('') }}"
# preserve the original string verbatim for all subsequent retries
target_version_full: "{{ target_version | default('') }}"
- name: Show received metadata
ansible.builtin.debug:
msg:
- "attempt={{ attempt }}"
- "max_attempts={{ effective_max_attempts }}"
- "correlation_id={{ correlation_id }}"
- "original_emitted_at={{ original_emitted_at }}"
- "target_version(full)={{ target_version_full }}"
# ---- Fast TCP reachability probe (controller-side) ----
- name: Check if TCP/{{ tcp_port }} is reachable with nc
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
register: nc_probe
changed_when: false
ignore_errors: true
- name: Build failure journal (no TCP connectivity) + mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (no TCP connectivity)
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
- name: Stop host after TCP failure (we’ll schedule or close out below)
when: nc_probe.rc != 0
ansible.builtin.meta: noop
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
- name: Probe banner via SSH from controller (classic extraction)
when: nc_probe.rc == 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
USER="{{ ssh_user }}"
HOST="{{ ansible_host | default(inventory_hostname) }}"
sshpass -p '{{ ssh_pass }}' \
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
"${USER}@${HOST}" \
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
register: banner_probe
changed_when: false
ignore_errors: true
- name: Show the current version (banner line)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.debug:
msg: "{{ banner_probe.stdout | trim }}"
- name: Evaluate version match (full-string contains check)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.set_fact:
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
# ---- Journaling paths ----
# Success: banner matches expected full target_version
- name: Build success journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_success_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish success journal to control queue
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_success_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_success
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
# NEW: send a control tag to clean up device state on success
- name: Build cleanup control payload (update_cleanup_success)
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
control_cleanup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "update_cleanup_success"
- name: Publish cleanup control message to control queue
when: control_cleanup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ control_cleanup_payload | to_json }}"
payload_encoding: "string"
register: rmq_cleanup_success
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
# Mismatch: reachable & banner read, but not equal to target_version
- name: Build mismatch journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_mismatch_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish mismatch journal to control queue
when: journal_mismatch_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_mismatch_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_mismatch
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
# SSH error path: TCP OK, but SSH failed
- name: Build failure journal payload (ssh error) + mark retry
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
journal_fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
{{ (banner_probe.stderr | default('') | trim) }}
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (ssh error) to control queue
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_fail
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
- name: Compute next-attempt delay (ms) according to policy
when: (_needs_retry | default(false)) | bool
ansible.builtin.set_fact:
next_attempt: "{{ attempt | int + 1 }}"
next_delay_sec: >-
{% if attempt | int == 1 %}
300
{% elif attempt | int == 2 %}
600
{% else %}
0
{% endif %}
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
# If we've reached the cap, send a final “gave up” journal and stop.
- name: Build final gave-up journal (max attempts reached)
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
journal_gaveup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
- name: Publish final gave-up journal
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_gaveup_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_gaveup
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
- name: Stop host after final gave-up
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
# Otherwise schedule the next attempt (only if we still have budget)
- name: Build delayed after-upgrade payload for next attempt
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
delayed_payload:
task_name: "afterupgrade_check"
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
attempt: "{{ next_attempt | int }}"
max_attempts: "{{ effective_max_attempts | int }}"
correlation_id: "{{ correlation_id }}"
original_emitted_at: "{{ original_emitted_at }}"
target_version: "{{ target_version_full }}"
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
expiration: "{{ (next_delay_ms | int) | string }}"
correlation_id: "{{ correlation_id }}"
routing_key: "deviceconfig"
payload: "{{ delayed_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_next
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
when: (_needs_retry | default(false)) | bool
ansible.builtin.meta: end_host