--- - name: Persuasive / hunting upgrade orchestrator hosts: all gather_facts: no vars: # RabbitMQ config rmq_host: "10.210.12.2" rmq_port: 15672 rmq_user: "admin" rmq_pass: "change_me" rmq_vhost: "app" # Exchanges / queues work_exchange: "deviceconfig" work_routing_key: "deviceconfig" # immediate work path holding_exchange: "deviceconfig.holding" holding_routing_key: "persuasive" # reschedules go to persuasive holding control_exchange: "controls" control_queue: "queue_controls" # Probing defaults tcp_port: 22 nc_timeout: 5 ssh_timeout: 10 ssh_user: "{{ ansible_user | default('root') }}" ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" # Policy defaults (overridable via -e from task_options) pu_period_default: "30m" pu_attempts_default: 48 pu_untilhours_default: "72h" tasks: - name: Normalize inputs (no self-referential defaults) ansible.builtin.set_fact: attempt: "{{ (attempt | default(1)) | int }}" pu_period: "{{ pu_period | default(pu_period_default) }}" pu_attempts: "{{ (pu_attempts | default(pu_attempts_default)) | int }}" pu_untilhours: "{{ pu_untilhours | default(pu_untilhours_default) }}" correlation_id: "{{ correlation_id | default('') }}" original_emitted_at: "{{ original_emitted_at | default(lookup('pipe','date -u +%FT%TZ')) }}" controller_now_iso: "{{ lookup('pipe','date -u +%FT%TZ') }}" - name: Show received metadata ansible.builtin.debug: msg: - "attempt={{ attempt }}" - "pu_period={{ pu_period }}" - "pu_attempts={{ pu_attempts }}" - "pu_untilhours={{ pu_untilhours }}" - "original_emitted_at={{ original_emitted_at }}" - "controller_now={{ controller_now_iso }}" # Optional time budget - name: Compute budget_ms (supports d/h/m/s in pu_untilhours) ansible.builtin.set_fact: budget_ms: >- {{ ( ( (pu_untilhours | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 + (pu_untilhours | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 + (pu_untilhours | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 + (pu_untilhours | regex_findall('([0-9]+)s') | first | default('0')) | int ) * 1000 ) }} - name: Compute elapsed since original emission (ms) ansible.builtin.set_fact: elapsed_ms: >- {{ ( (lookup('pipe', 'date -u -d ' ~ controller_now_iso ~ ' +%s') | int) - (lookup('pipe', 'date -u -d ' ~ original_emitted_at ~ ' +%s') | int) ) * 1000 }} # Helpers for final summary line - name: Compute period_sec from pu_period (supports d/h/m/s) ansible.builtin.set_fact: period_sec: >- {{ ( (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 + (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 + (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 + (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int ) }} - name: Compute budget_hours string (one decimal) ansible.builtin.set_fact: budget_hours_str: "{{ '%.1f' | format( (budget_ms | int) / 3600000.0 ) }}" - name: Gave up due to time budget when: (elapsed_ms | int) >= (budget_ms | int) delegate_to: localhost block: - name: Journal final give-up (unified message) ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: { content-type: "application/json" } body_format: json body: properties: { content_type: "application/json" } routing_key: "{{ control_queue }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "journal_add", "task_result": ( "device wasn't reachable for " ~ (attempt | string) ~ " attempts, each " ~ (period_sec | string) ~ " seconds for " ~ budget_hours_str ~ " hours. backing off, won't persuade it more. Schedule again if needed (limit: time budget exhausted)" ) } | to_json }} payload_encoding: "string" register: rmq_j_budget changed_when: (rmq_j_budget.json is defined) and (rmq_j_budget.json.routed | default(false) | bool) - ansible.builtin.meta: end_host # Attempt cap - name: Gave up due to attempts cap when: (attempt | int) >= (pu_attempts | int) delegate_to: localhost block: - name: Journal final give-up (unified message) ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: { content-type: "application/json" } body_format: json body: properties: { content_type: "application/json" } routing_key: "{{ control_queue }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "journal_add", "task_result": ( "device wasn't reachable for " ~ (attempt | string) ~ " attempts, each " ~ (period_sec | string) ~ " seconds for " ~ budget_hours_str ~ " hours. backing off, won't persuade it more. Schedule again if needed (limit: attempts cap reached)" ) } | to_json }} payload_encoding: "string" register: rmq_j_gaveup changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) - ansible.builtin.meta: end_host # Probe - name: Check TCP/{{ tcp_port }} via nc delegate_to: localhost ansible.builtin.shell: | nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} register: nc_probe changed_when: false ignore_errors: true # ONLINE → hand off to normal path - name: Publish upgrade-confirmed to deviceconfig (immediate) when: nc_probe.rc == 0 delegate_to: localhost ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ work_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: { content-type: "application/json" } body_format: json body: properties: { content_type: "application/json" } routing_key: "{{ work_routing_key }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "upgrade-confirmed" } | to_json }} payload_encoding: "string" register: rmq_start changed_when: (rmq_start.json is defined) and (rmq_start.json.routed | default(false) | bool) - name: "Journal: online, starting upgrade" when: nc_probe.rc == 0 delegate_to: localhost ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: { content-type: "application/json" } body_format: json body: properties: { content_type: "application/json" } routing_key: "{{ control_queue }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "journal_add", "task_result": ("persuasive-upgrade: device online, starting upgrade-confirmed now. Original=" ~ original_emitted_at) } | to_json }} payload_encoding: "string" register: rmq_j_start changed_when: (rmq_j_start.json is defined) and (rmq_j_start.json.routed | default(false) | bool) - ansible.builtin.meta: end_host when: nc_probe.rc == 0 # OFFLINE → reschedule into persuasive holding - name: Compute TTL ms from pu_period (supports d/h/m/s) when: nc_probe.rc != 0 ansible.builtin.set_fact: ttl_ms: >- {{ ( ( (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 + (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 + (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 + (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int ) * 1000 ) }} - name: Compute next_attempt when: nc_probe.rc != 0 ansible.builtin.set_fact: next_attempt: "{{ attempt | int + 1 }}" - name: Build next task_options (carry policy + increment attempt) when: nc_probe.rc != 0 ansible.builtin.set_fact: next_task_options: >- -e pu_period={{ pu_period }} -e pu_attempts={{ pu_attempts }} -e pu_untilhours={{ pu_untilhours }} -e attempt={{ next_attempt }} -e original_emitted_at='{{ original_emitted_at }}' - name: Decide if we should emit the reschedule journal this attempt when: nc_probe.rc != 0 ansible.builtin.set_fact: _pu_journal_this_try: "{{ (attempt | int) in [1, 2] or ((attempt | int) % 10 == 0) }}" - name: "Publish delayed persuasive-upgrade to holding (routing: persuasive)" when: nc_probe.rc != 0 delegate_to: localhost ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: { content-type: "application/json" } body_format: json body: properties: content_type: "application/json" headers: x-delay: "{{ ttl_ms | int }}" routing_key: "{{ holding_routing_key }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "persuasive-upgrade", "task_options": (next_task_options | trim) } | to_json }} payload_encoding: "string" register: rmq_pub_next changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) - name: "Journal: offline, rescheduled" when: nc_probe.rc != 0 and (_pu_journal_this_try | bool) delegate_to: localhost ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: { content-type: "application/json" } body_format: json body: properties: { content_type: "application/json" } routing_key: "{{ control_queue }}" payload: >- {{ { "inscope_device": (ansible_hostname | default(inventory_hostname)), "task_name": "journal_add", "task_result": ("persuasive-upgrade: device offline, rescheduling (attempt " ~ attempt ~ "/" ~ pu_attempts ~ ", next in " ~ pu_period ~ "). Original=" ~ original_emitted_at) } | to_json }} payload_encoding: "string" register: rmq_j_off changed_when: (rmq_j_off.json is defined) and (rmq_j_off.json.routed | default(false) | bool) - ansible.builtin.meta: end_host when: nc_probe.rc != 0