331 lines
13 KiB
YAML
331 lines
13 KiB
YAML
---
|
|
- name: Persuasive / hunting upgrade orchestrator
|
|
hosts: all
|
|
gather_facts: no
|
|
|
|
vars:
|
|
# RabbitMQ config
|
|
rmq_host: "10.210.12.2"
|
|
rmq_port: 15672
|
|
rmq_user: "admin"
|
|
rmq_pass: "change_me"
|
|
rmq_vhost: "app"
|
|
|
|
# Exchanges / queues
|
|
work_exchange: "deviceconfig"
|
|
work_routing_key: "deviceconfig" # immediate work path
|
|
holding_exchange: "deviceconfig.holding"
|
|
holding_routing_key: "persuasive" # reschedules go to persuasive holding
|
|
control_exchange: "controls"
|
|
control_queue: "queue_controls"
|
|
|
|
# Probing defaults
|
|
tcp_port: 22
|
|
nc_timeout: 5
|
|
ssh_timeout: 10
|
|
ssh_user: "{{ ansible_user | default('root') }}"
|
|
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
|
|
|
|
# Policy defaults (overridable via -e from task_options)
|
|
pu_period_default: "30m"
|
|
pu_attempts_default: 48
|
|
pu_untilhours_default: "72h"
|
|
|
|
tasks:
|
|
- name: Normalize inputs (no self-referential defaults)
|
|
ansible.builtin.set_fact:
|
|
attempt: "{{ (attempt | default(1)) | int }}"
|
|
pu_period: "{{ pu_period | default(pu_period_default) }}"
|
|
pu_attempts: "{{ (pu_attempts | default(pu_attempts_default)) | int }}"
|
|
pu_untilhours: "{{ pu_untilhours | default(pu_untilhours_default) }}"
|
|
correlation_id: "{{ correlation_id | default('') }}"
|
|
original_emitted_at: "{{ original_emitted_at | default(lookup('pipe','date -u +%FT%TZ')) }}"
|
|
controller_now_iso: "{{ lookup('pipe','date -u +%FT%TZ') }}"
|
|
|
|
- name: Show received metadata
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "attempt={{ attempt }}"
|
|
- "pu_period={{ pu_period }}"
|
|
- "pu_attempts={{ pu_attempts }}"
|
|
- "pu_untilhours={{ pu_untilhours }}"
|
|
- "original_emitted_at={{ original_emitted_at }}"
|
|
- "controller_now={{ controller_now_iso }}"
|
|
|
|
# Optional time budget
|
|
- name: Compute budget_ms (supports d/h/m/s in pu_untilhours)
|
|
ansible.builtin.set_fact:
|
|
budget_ms: >-
|
|
{{
|
|
(
|
|
(
|
|
(pu_untilhours | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
|
|
(pu_untilhours | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
|
|
(pu_untilhours | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
|
|
(pu_untilhours | regex_findall('([0-9]+)s') | first | default('0')) | int
|
|
) * 1000
|
|
)
|
|
}}
|
|
|
|
- name: Compute elapsed since original emission (ms)
|
|
ansible.builtin.set_fact:
|
|
elapsed_ms: >-
|
|
{{
|
|
(
|
|
(lookup('pipe', 'date -u -d ' ~ controller_now_iso ~ ' +%s') | int) -
|
|
(lookup('pipe', 'date -u -d ' ~ original_emitted_at ~ ' +%s') | int)
|
|
) * 1000
|
|
}}
|
|
|
|
# Helpers for final summary line
|
|
- name: Compute period_sec from pu_period (supports d/h/m/s)
|
|
ansible.builtin.set_fact:
|
|
period_sec: >-
|
|
{{
|
|
(
|
|
(pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
|
|
(pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
|
|
(pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
|
|
(pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
|
|
)
|
|
}}
|
|
|
|
- name: Compute budget_hours string (one decimal)
|
|
ansible.builtin.set_fact:
|
|
budget_hours_str: "{{ '%.1f' | format( (budget_ms | int) / 3600000.0 ) }}"
|
|
|
|
- name: Gave up due to time budget
|
|
when: (elapsed_ms | int) >= (budget_ms | int)
|
|
delegate_to: localhost
|
|
block:
|
|
- name: Journal final give-up (unified message)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "journal_add",
|
|
"task_result": (
|
|
"device wasn't reachable for " ~ (attempt | string) ~
|
|
" attempts, each " ~ (period_sec | string) ~
|
|
" seconds for " ~ budget_hours_str ~
|
|
" hours. backing off, won't persuade it more. Schedule again if needed (limit: time budget exhausted)"
|
|
)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_j_budget
|
|
changed_when: (rmq_j_budget.json is defined) and (rmq_j_budget.json.routed | default(false) | bool)
|
|
|
|
- ansible.builtin.meta: end_host
|
|
|
|
# Attempt cap
|
|
- name: Gave up due to attempts cap
|
|
when: (attempt | int) >= (pu_attempts | int)
|
|
delegate_to: localhost
|
|
block:
|
|
- name: Journal final give-up (unified message)
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "journal_add",
|
|
"task_result": (
|
|
"device wasn't reachable for " ~ (attempt | string) ~
|
|
" attempts, each " ~ (period_sec | string) ~
|
|
" seconds for " ~ budget_hours_str ~
|
|
" hours. backing off, won't persuade it more. Schedule again if needed (limit: attempts cap reached)"
|
|
)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_j_gaveup
|
|
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
|
|
|
- ansible.builtin.meta: end_host
|
|
|
|
# Probe
|
|
- name: Check TCP/{{ tcp_port }} via nc
|
|
delegate_to: localhost
|
|
ansible.builtin.shell: |
|
|
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
|
|
register: nc_probe
|
|
changed_when: false
|
|
ignore_errors: true
|
|
|
|
# ONLINE → hand off to normal path
|
|
- name: Publish upgrade-confirmed to deviceconfig (immediate)
|
|
when: nc_probe.rc == 0
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ work_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ work_routing_key }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "upgrade-confirmed"
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_start
|
|
changed_when: (rmq_start.json is defined) and (rmq_start.json.routed | default(false) | bool)
|
|
|
|
- name: "Journal: online, starting upgrade"
|
|
when: nc_probe.rc == 0
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "journal_add",
|
|
"task_result": ("persuasive-upgrade: device online, starting upgrade-confirmed now. Original=" ~ original_emitted_at)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_j_start
|
|
changed_when: (rmq_j_start.json is defined) and (rmq_j_start.json.routed | default(false) | bool)
|
|
|
|
- ansible.builtin.meta: end_host
|
|
when: nc_probe.rc == 0
|
|
|
|
# OFFLINE → reschedule into persuasive holding
|
|
- name: Compute TTL ms from pu_period (supports d/h/m/s)
|
|
when: nc_probe.rc != 0
|
|
ansible.builtin.set_fact:
|
|
ttl_ms: >-
|
|
{{
|
|
(
|
|
(
|
|
(pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 +
|
|
(pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 +
|
|
(pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 +
|
|
(pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int
|
|
) * 1000
|
|
)
|
|
}}
|
|
|
|
- name: Compute next_attempt
|
|
when: nc_probe.rc != 0
|
|
ansible.builtin.set_fact:
|
|
next_attempt: "{{ attempt | int + 1 }}"
|
|
|
|
- name: Build next task_options (carry policy + increment attempt)
|
|
when: nc_probe.rc != 0
|
|
ansible.builtin.set_fact:
|
|
next_task_options: >-
|
|
-e pu_period={{ pu_period }}
|
|
-e pu_attempts={{ pu_attempts }}
|
|
-e pu_untilhours={{ pu_untilhours }}
|
|
-e attempt={{ next_attempt }}
|
|
-e original_emitted_at='{{ original_emitted_at }}'
|
|
|
|
- name: Decide if we should emit the reschedule journal this attempt
|
|
when: nc_probe.rc != 0
|
|
ansible.builtin.set_fact:
|
|
_pu_journal_this_try: "{{ (attempt | int) in [1, 2] or ((attempt | int) % 10 == 0) }}"
|
|
|
|
- name: "Publish delayed persuasive-upgrade to holding (routing: persuasive)"
|
|
when: nc_probe.rc != 0
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
headers:
|
|
x-delay: "{{ ttl_ms | int }}"
|
|
routing_key: "{{ holding_routing_key }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "persuasive-upgrade",
|
|
"task_options": (next_task_options | trim)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_pub_next
|
|
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
|
|
- name: "Journal: offline, rescheduled"
|
|
when: nc_probe.rc != 0 and (_pu_journal_this_try | bool)
|
|
delegate_to: localhost
|
|
ansible.builtin.uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: >-
|
|
{{
|
|
{
|
|
"inscope_device": (ansible_hostname | default(inventory_hostname)),
|
|
"task_name": "journal_add",
|
|
"task_result": ("persuasive-upgrade: device offline, rescheduling (attempt " ~ attempt ~ "/" ~ pu_attempts ~ ", next in " ~ pu_period ~ "). Original=" ~ original_emitted_at)
|
|
} | to_json
|
|
}}
|
|
payload_encoding: "string"
|
|
register: rmq_j_off
|
|
changed_when: (rmq_j_off.json is defined) and (rmq_j_off.json.routed | default(false) | bool)
|
|
|
|
- ansible.builtin.meta: end_host
|
|
when: nc_probe.rc != 0
|