first commit
This commit is contained in:
ansible user
330
files/ansible-playbooks/persuasive-upgrade.yml
Normal file
330
files/ansible-playbooks/persuasive-upgrade.yml
Normal file
@@ -0,0 +1,330 @@
|
||||
---
|
||||
- name: Persuasive / hunting upgrade orchestrator
|
||||
hosts: all
|
||||
gather_facts: no
|
||||
|
||||
vars:
|
||||
# RabbitMQ config
|
||||
rmq_host: "10.210.12.2"
|
||||
rmq_port: 15672
|
||||
rmq_user: "admin"
|
||||
rmq_pass: "change_me"
|
||||
rmq_vhost: "app"
|
||||
|
||||
# Exchanges / queues
|
||||
work_exchange: "deviceconfig"
|
||||
work_routing_key: "deviceconfig" # immediate work path
|
||||
holding_exchange: "deviceconfig.holding"
|
||||
holding_routing_key: "persuasive" # reschedules go to persuasive holding
|
||||
control_exchange: "controls"
|
||||
control_queue: "queue_controls"
|
||||
|
||||
# Probing defaults
|
||||
tcp_port: 22
|
||||
nc_timeout: 5
|
||||
ssh_timeout: 10
|
||||
ssh_user: "{{ ansible_user | default('root') }}"
|
||||
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
|
||||
|
||||
# Policy defaults (overridable via -e from task_options)
|
||||
pu_period_default: "30m"
|
||||
pu_attempts_default: 48
|
||||
pu_untilhours_default: "72h"
|
||||
|
||||
tasks:
|
||||
- name: Normalize inputs (no self-referential defaults)
|
||||
ansible.builtin.set_fact:
|
||||
attempt: "{{ (attempt | default(1)) | int }}"
|
||||
pu_period: "{{ pu_period | default(pu_period_default) }}"
|
||||
pu_attempts: "{{ (pu_attempts | default(pu_attempts_default)) | int }}"
|
||||
pu_untilhours: "{{ pu_untilhours | default(pu_untilhours_default) }}"
|
||||
correlation_id: "{{ correlation_id | default('') }}"
|
||||
original_emitted_at: "{{ original_emitted_at | default(lookup('pipe','date -u +%FT%TZ')) }}"
|
||||
controller_now_iso: "{{ lookup('pipe','date -u +%FT%TZ') }}"
|
||||
|
||||
- name: Show received metadata
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "attempt={{ attempt }}"
|
||||
- "pu_period={{ pu_period }}"
|
||||
- "pu_attempts={{ pu_attempts }}"
|
||||
- "pu_untilhours={{ pu_untilhours }}"
|
||||
- "original_emitted_at={{ original_emitted_at }}"
|
||||
- "controller_now={{ controller_now_iso }}"
|
||||
|
||||
# Optional time budget
|
||||
- name: Compute budget_ms (supports d/h/m/s in pu_untilhours)
|
||||
ansible.builtin.set_fact:
|
||||
budget_ms: >-
|
||||
{{
|
||||
(
|
||||
(
|
||||
(pu_untilhours | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
|
||||
(pu_untilhours | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
|
||||
(pu_untilhours | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
|
||||
(pu_untilhours | regex_findall('([0-9]+)s') | first | default('0')) | int
|
||||
) * 1000
|
||||
)
|
||||
}}
|
||||
|
||||
- name: Compute elapsed since original emission (ms)
|
||||
ansible.builtin.set_fact:
|
||||
elapsed_ms: >-
|
||||
{{
|
||||
(
|
||||
(lookup('pipe', 'date -u -d ' ~ controller_now_iso ~ ' +%s') | int) -
|
||||
(lookup('pipe', 'date -u -d ' ~ original_emitted_at ~ ' +%s') | int)
|
||||
) * 1000
|
||||
}}
|
||||
|
||||
# Helpers for final summary line
|
||||
- name: Compute period_sec from pu_period (supports d/h/m/s)
|
||||
ansible.builtin.set_fact:
|
||||
period_sec: >-
|
||||
{{
|
||||
(
|
||||
(pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
|
||||
(pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
|
||||
(pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
|
||||
(pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
|
||||
)
|
||||
}}
|
||||
|
||||
- name: Compute budget_hours string (one decimal)
|
||||
ansible.builtin.set_fact:
|
||||
budget_hours_str: "{{ '%.1f' | format( (budget_ms | int) / 3600000.0 ) }}"
|
||||
|
||||
- name: Gave up due to time budget
|
||||
when: (elapsed_ms | int) >= (budget_ms | int)
|
||||
delegate_to: localhost
|
||||
block:
|
||||
- name: Journal final give-up (unified message)
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
||||
method: POST
|
||||
user: "{{ rmq_user }}"
|
||||
password: "{{ rmq_pass }}"
|
||||
force_basic_auth: true
|
||||
status_code: 200
|
||||
headers: { content-type: "application/json" }
|
||||
body_format: json
|
||||
body:
|
||||
properties: { content_type: "application/json" }
|
||||
routing_key: "{{ control_queue }}"
|
||||
payload: >-
|
||||
{{
|
||||
{
|
||||
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
||||
"task_name": "journal_add",
|
||||
"task_result": (
|
||||
"device wasn't reachable for " ~ (attempt | string) ~
|
||||
" attempts, each " ~ (period_sec | string) ~
|
||||
" seconds for " ~ budget_hours_str ~
|
||||
" hours. backing off, won't persuade it more. Schedule again if needed (limit: time budget exhausted)"
|
||||
)
|
||||
} | to_json
|
||||
}}
|
||||
payload_encoding: "string"
|
||||
register: rmq_j_budget
|
||||
changed_when: (rmq_j_budget.json is defined) and (rmq_j_budget.json.routed | default(false) | bool)
|
||||
|
||||
- ansible.builtin.meta: end_host
|
||||
|
||||
# Attempt cap
|
||||
- name: Gave up due to attempts cap
|
||||
when: (attempt | int) >= (pu_attempts | int)
|
||||
delegate_to: localhost
|
||||
block:
|
||||
- name: Journal final give-up (unified message)
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
||||
method: POST
|
||||
user: "{{ rmq_user }}"
|
||||
password: "{{ rmq_pass }}"
|
||||
force_basic_auth: true
|
||||
status_code: 200
|
||||
headers: { content-type: "application/json" }
|
||||
body_format: json
|
||||
body:
|
||||
properties: { content_type: "application/json" }
|
||||
routing_key: "{{ control_queue }}"
|
||||
payload: >-
|
||||
{{
|
||||
{
|
||||
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
||||
"task_name": "journal_add",
|
||||
"task_result": (
|
||||
"device wasn't reachable for " ~ (attempt | string) ~
|
||||
" attempts, each " ~ (period_sec | string) ~
|
||||
" seconds for " ~ budget_hours_str ~
|
||||
" hours. backing off, won't persuade it more. Schedule again if needed (limit: attempts cap reached)"
|
||||
)
|
||||
} | to_json
|
||||
}}
|
||||
payload_encoding: "string"
|
||||
register: rmq_j_gaveup
|
||||
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
||||
|
||||
- ansible.builtin.meta: end_host
|
||||
|
||||
# Probe
|
||||
- name: Check TCP/{{ tcp_port }} via nc
|
||||
delegate_to: localhost
|
||||
ansible.builtin.shell: |
|
||||
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
|
||||
register: nc_probe
|
||||
changed_when: false
|
||||
ignore_errors: true
|
||||
|
||||
# ONLINE → hand off to normal path
|
||||
- name: Publish upgrade-confirmed to deviceconfig (immediate)
|
||||
when: nc_probe.rc == 0
|
||||
delegate_to: localhost
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ work_exchange | urlencode }}/publish"
|
||||
method: POST
|
||||
user: "{{ rmq_user }}"
|
||||
password: "{{ rmq_pass }}"
|
||||
force_basic_auth: true
|
||||
status_code: 200
|
||||
headers: { content-type: "application/json" }
|
||||
body_format: json
|
||||
body:
|
||||
properties: { content_type: "application/json" }
|
||||
routing_key: "{{ work_routing_key }}"
|
||||
payload: >-
|
||||
{{
|
||||
{
|
||||
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
||||
"task_name": "upgrade-confirmed"
|
||||
} | to_json
|
||||
}}
|
||||
payload_encoding: "string"
|
||||
register: rmq_start
|
||||
changed_when: (rmq_start.json is defined) and (rmq_start.json.routed | default(false) | bool)
|
||||
|
||||
- name: "Journal: online, starting upgrade"
|
||||
when: nc_probe.rc == 0
|
||||
delegate_to: localhost
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
||||
method: POST
|
||||
user: "{{ rmq_user }}"
|
||||
password: "{{ rmq_pass }}"
|
||||
force_basic_auth: true
|
||||
status_code: 200
|
||||
headers: { content-type: "application/json" }
|
||||
body_format: json
|
||||
body:
|
||||
properties: { content_type: "application/json" }
|
||||
routing_key: "{{ control_queue }}"
|
||||
payload: >-
|
||||
{{
|
||||
{
|
||||
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
||||
"task_name": "journal_add",
|
||||
"task_result": ("persuasive-upgrade: device online, starting upgrade-confirmed now. Original=" ~ original_emitted_at)
|
||||
} | to_json
|
||||
}}
|
||||
payload_encoding: "string"
|
||||
register: rmq_j_start
|
||||
changed_when: (rmq_j_start.json is defined) and (rmq_j_start.json.routed | default(false) | bool)
|
||||
|
||||
- ansible.builtin.meta: end_host
|
||||
when: nc_probe.rc == 0
|
||||
|
||||
# OFFLINE → reschedule into persuasive holding
|
||||
- name: Compute TTL ms from pu_period (supports d/h/m/s)
|
||||
when: nc_probe.rc != 0
|
||||
ansible.builtin.set_fact:
|
||||
ttl_ms: >-
|
||||
{{
|
||||
(
|
||||
(
|
||||
(pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
|
||||
(pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
|
||||
(pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
|
||||
(pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
|
||||
) * 1000
|
||||
)
|
||||
}}
|
||||
|
||||
- name: Compute next_attempt
|
||||
when: nc_probe.rc != 0
|
||||
ansible.builtin.set_fact:
|
||||
next_attempt: "{{ attempt | int + 1 }}"
|
||||
|
||||
- name: Build next task_options (carry policy + increment attempt)
|
||||
when: nc_probe.rc != 0
|
||||
ansible.builtin.set_fact:
|
||||
next_task_options: >-
|
||||
-e pu_period={{ pu_period }}
|
||||
-e pu_attempts={{ pu_attempts }}
|
||||
-e pu_untilhours={{ pu_untilhours }}
|
||||
-e attempt={{ next_attempt }}
|
||||
-e original_emitted_at='{{ original_emitted_at }}'
|
||||
|
||||
- name: Decide if we should emit the reschedule journal this attempt
|
||||
when: nc_probe.rc != 0
|
||||
ansible.builtin.set_fact:
|
||||
_pu_journal_this_try: "{{ (attempt | int) in [1, 2] or ((attempt | int) % 10 == 0) }}"
|
||||
|
||||
- name: "Publish delayed persuasive-upgrade to holding (routing: persuasive)"
|
||||
when: nc_probe.rc != 0
|
||||
delegate_to: localhost
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
|
||||
method: POST
|
||||
user: "{{ rmq_user }}"
|
||||
password: "{{ rmq_pass }}"
|
||||
force_basic_auth: true
|
||||
status_code: 200
|
||||
headers: { content-type: "application/json" }
|
||||
body_format: json
|
||||
body:
|
||||
properties:
|
||||
content_type: "application/json"
|
||||
headers:
|
||||
x-delay: "{{ ttl_ms | int }}"
|
||||
routing_key: "{{ holding_routing_key }}"
|
||||
payload: >-
|
||||
{{
|
||||
{
|
||||
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
||||
"task_name": "persuasive-upgrade",
|
||||
"task_options": (next_task_options | trim)
|
||||
} | to_json
|
||||
}}
|
||||
payload_encoding: "string"
|
||||
register: rmq_pub_next
|
||||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
||||
|
||||
- name: "Journal: offline, rescheduled"
|
||||
when: nc_probe.rc != 0 and (_pu_journal_this_try | bool)
|
||||
delegate_to: localhost
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
||||
method: POST
|
||||
user: "{{ rmq_user }}"
|
||||
password: "{{ rmq_pass }}"
|
||||
force_basic_auth: true
|
||||
status_code: 200
|
||||
headers: { content-type: "application/json" }
|
||||
body_format: json
|
||||
body:
|
||||
properties: { content_type: "application/json" }
|
||||
routing_key: "{{ control_queue }}"
|
||||
payload: >-
|
||||
{{
|
||||
{
|
||||
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
||||
"task_name": "journal_add",
|
||||
"task_result": ("persuasive-upgrade: device offline, rescheduling (attempt " ~ attempt ~ "/" ~ pu_attempts ~ ", next in " ~ pu_period ~ "). Original=" ~ original_emitted_at)
|
||||
} | to_json
|
||||
}}
|
||||
payload_encoding: "string"
|
||||
register: rmq_j_off
|
||||
changed_when: (rmq_j_off.json is defined) and (rmq_j_off.json.routed | default(false) | bool)
|
||||
|
||||
- ansible.builtin.meta: end_host
|
||||
when: nc_probe.rc != 0
|
||||
Reference in New Issue
Block a user