0509
This commit is contained in:
@@ -1,355 +0,0 @@
|
|||||||
---
|
|
||||||
- name: After-upgrade verification (banner check + reporting)
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
# RabbitMQ + defaults (match the big script)
|
|
||||||
vars:
|
|
||||||
rmq_host: "10.210.12.2"
|
|
||||||
rmq_port: 15672
|
|
||||||
rmq_user: "admin"
|
|
||||||
rmq_pass: "change_me"
|
|
||||||
rmq_vhost: "app"
|
|
||||||
rmq_exchange: "controls"
|
|
||||||
control_queue: "queue_controls"
|
|
||||||
|
|
||||||
# Probing/SSH defaults
|
|
||||||
tcp_port: 22
|
|
||||||
nc_timeout: 5
|
|
||||||
ssh_user: "{{ ansible_user | default('root') }}"
|
|
||||||
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
|
|
||||||
ssh_timeout: 10
|
|
||||||
|
|
||||||
# Do NOT self-reference max_attempts. We’ll normalize below.
|
|
||||||
max_attempts_default: 3
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# ---- Normalize metadata safely (no self-referential defaults) ----
|
|
||||||
- name: Normalize metadata (no clever transforms)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
attempt: "{{ (attempt | default(1)) | int }}"
|
|
||||||
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
||||||
target_version: "{{ target_version | default('') }}"
|
|
||||||
# preserve the original string verbatim for all subsequent retries
|
|
||||||
target_version_full: "{{ target_version | default('') }}"
|
|
||||||
|
|
||||||
- name: Show received metadata
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "attempt={{ attempt }}"
|
|
||||||
- "max_attempts={{ effective_max_attempts }}"
|
|
||||||
- "correlation_id={{ correlation_id }}"
|
|
||||||
- "original_emitted_at={{ original_emitted_at }}"
|
|
||||||
- "target_version(full)={{ target_version_full }}"
|
|
||||||
|
|
||||||
# ---- Fast TCP reachability probe (controller-side) ----
|
|
||||||
- name: Check if TCP/{{ tcp_port }} is reachable with nc
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
|
|
||||||
register: nc_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Build failure journal (no TCP connectivity) + mark retry
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (no TCP connectivity)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_tcp_fail
|
|
||||||
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
|
|
||||||
- name: Stop host after TCP failure (we’ll schedule or close out below)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
ansible.builtin.meta: noop
|
|
||||||
|
|
||||||
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
|
|
||||||
- name: Probe banner via SSH from controller (classic extraction)
|
|
||||||
when: nc_probe.rc == 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
USER="{{ ssh_user }}"
|
|
||||||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
sshpass -p '{{ ssh_pass }}' \
|
|
||||||
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
|
|
||||||
"${USER}@${HOST}" \
|
|
||||||
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
|
|
||||||
register: banner_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Show the current version (banner line)
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "{{ banner_probe.stdout | trim }}"
|
|
||||||
|
|
||||||
- name: Evaluate version match (full-string contains check)
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
|
|
||||||
|
|
||||||
# ---- Journaling paths ----
|
|
||||||
# Success: banner matches expected full target_version
|
|
||||||
- name: Build success journal payload
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_success_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
|
||||||
|
|
||||||
- name: Publish success journal to control queue
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_success_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_success
|
|
||||||
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# NEW: send a control tag to clean up device state on success
|
|
||||||
- name: Build cleanup control payload (update_cleanup_success)
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
control_cleanup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "update_cleanup_success"
|
|
||||||
|
|
||||||
- name: Publish cleanup control message to control queue
|
|
||||||
when: control_cleanup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ control_cleanup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_cleanup_success
|
|
||||||
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# Mismatch: reachable & banner read, but not equal to target_version
|
|
||||||
- name: Build mismatch journal payload
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_mismatch_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
|
||||||
|
|
||||||
- name: Publish mismatch journal to control queue
|
|
||||||
when: journal_mismatch_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_mismatch_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_mismatch
|
|
||||||
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# SSH error path: TCP OK, but SSH failed
|
|
||||||
- name: Build failure journal payload (ssh error) + mark retry
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
{{ (banner_probe.stderr | default('') | trim) }}
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (ssh error) to control queue
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_fail
|
|
||||||
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
|
|
||||||
- name: Compute next-attempt delay (ms) according to policy
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
next_attempt: "{{ attempt | int + 1 }}"
|
|
||||||
next_delay_sec: >-
|
|
||||||
{% if attempt | int == 1 %}
|
|
||||||
300
|
|
||||||
{% elif attempt | int == 2 %}
|
|
||||||
600
|
|
||||||
{% else %}
|
|
||||||
0
|
|
||||||
{% endif %}
|
|
||||||
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
|
|
||||||
|
|
||||||
# If we've reached the cap, send a final “gave up” journal and stop.
|
|
||||||
- name: Build final gave-up journal (max attempts reached)
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_gaveup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
|
|
||||||
- name: Publish final gave-up journal
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_gaveup
|
|
||||||
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after final gave-up
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
# Otherwise schedule the next attempt (only if we still have budget)
|
|
||||||
- name: Build delayed after-upgrade payload for next attempt
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
delayed_payload:
|
|
||||||
task_name: "afterupgrade_check"
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
attempt: "{{ next_attempt | int }}"
|
|
||||||
max_attempts: "{{ effective_max_attempts | int }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at }}"
|
|
||||||
target_version: "{{ target_version_full }}"
|
|
||||||
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
||||||
schema_version: 1
|
|
||||||
|
|
||||||
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
|
|
||||||
when: delayed_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
routing_key: "deviceconfig"
|
|
||||||
payload: "{{ delayed_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_next
|
|
||||||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
@@ -1,432 +0,0 @@
|
|||||||
---
|
|
||||||
- name: After-upgrade verification (banner check + reporting)
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
# RabbitMQ + defaults (match the big script)
|
|
||||||
vars:
|
|
||||||
rmq_host: "10.210.12.2"
|
|
||||||
rmq_port: 15672
|
|
||||||
rmq_user: "admin"
|
|
||||||
rmq_pass: "change_me"
|
|
||||||
rmq_vhost: "app"
|
|
||||||
rmq_exchange: "controls"
|
|
||||||
control_queue: "queue_controls"
|
|
||||||
|
|
||||||
# Probing/SSH defaults
|
|
||||||
tcp_port: 22
|
|
||||||
nc_timeout: 5
|
|
||||||
ssh_user: "{{ ansible_user | default('root') }}"
|
|
||||||
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
|
|
||||||
ssh_timeout: 10
|
|
||||||
|
|
||||||
# Do NOT self-reference max_attempts. We’ll normalize below.
|
|
||||||
max_attempts_default: 3
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# ---- Normalize metadata safely (no self-referential defaults) ----
|
|
||||||
- name: Normalize metadata (no clever transforms)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
attempt: "{{ (attempt | default(1)) | int }}"
|
|
||||||
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
||||||
target_version: "{{ target_version | default('') }}"
|
|
||||||
# preserve the original string verbatim for all subsequent retries
|
|
||||||
target_version_full: "{{ target_version | default('') }}"
|
|
||||||
|
|
||||||
- name: Show received metadata
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "attempt={{ attempt }}"
|
|
||||||
- "max_attempts={{ effective_max_attempts }}"
|
|
||||||
- "correlation_id={{ correlation_id }}"
|
|
||||||
- "original_emitted_at={{ original_emitted_at }}"
|
|
||||||
- "target_version(full)={{ target_version_full }}"
|
|
||||||
|
|
||||||
# ---- Fast TCP reachability probe (controller-side) ----
|
|
||||||
- name: Check if TCP/{{ tcp_port }} is reachable with nc
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
|
|
||||||
register: nc_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Build failure journal (no TCP connectivity) + mark retry
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
|
|
||||||
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (no TCP connectivity)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_tcp_fail
|
|
||||||
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
|
|
||||||
- name: Stop host after TCP failure (we’ll schedule or close out below)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
ansible.builtin.meta: noop
|
|
||||||
|
|
||||||
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
|
|
||||||
- name: Probe banner via SSH from controller (classic extraction)
|
|
||||||
when: nc_probe.rc == 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
USER="{{ ssh_user }}"
|
|
||||||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
sshpass -p '{{ ssh_pass }}' \
|
|
||||||
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
|
|
||||||
"${USER}@${HOST}" \
|
|
||||||
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
|
|
||||||
register: banner_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Show the current version (banner line)
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "{{ banner_probe.stdout | trim }}"
|
|
||||||
|
|
||||||
- name: Evaluate version match (full-string contains check)
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
|
|
||||||
|
|
||||||
# ---- Journaling paths ----
|
|
||||||
|
|
||||||
# Success: banner matches expected full target_version
|
|
||||||
- name: Build success journal payload
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_success_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Banner='{{ (banner_probe.stdout | default('') | trim) }}'
|
|
||||||
Target='{{ target_version_full }}'
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
|
||||||
|
|
||||||
- name: Publish success journal to control queue
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_success_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_success
|
|
||||||
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# NEW: send a control tag to clean up device state on success
|
|
||||||
- name: Build cleanup control payload (update_cleanup_success)
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
control_cleanup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "update_cleanup_success"
|
|
||||||
|
|
||||||
- name: Publish cleanup control message to control queue
|
|
||||||
when: control_cleanup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ control_cleanup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_cleanup_success
|
|
||||||
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag ---
|
|
||||||
|
|
||||||
- name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801")
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
indoor_fwver_norm: >-
|
|
||||||
{{
|
|
||||||
( (dev2_fwver.stdout | default('') | trim)
|
|
||||||
| regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1')
|
|
||||||
| regex_replace('\\s+', ' ')
|
|
||||||
| trim
|
|
||||||
)
|
|
||||||
}}
|
|
||||||
|
|
||||||
- name: Publish NetBox custom field indoor_fwver to control queue
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers: { content-type: "application/json" }
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties: { content_type: "application/json" }
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ {
|
|
||||||
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
||||||
'task_name': 'custom_field_set',
|
|
||||||
'task_add1': 'indoor_fwver',
|
|
||||||
'task_result': (indoor_fwver_norm | default(''))
|
|
||||||
} | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_cf_indoor_fwver
|
|
||||||
changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Build payload to remove indoor-restart-scheduled tag (DEV1)
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
tag_remove_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "tag_remove"
|
|
||||||
task_result: "indoor-restart-scheduled"
|
|
||||||
|
|
||||||
- name: Publish indoor-restart-scheduled tag removal
|
|
||||||
when: tag_remove_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ tag_remove_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_tag_remove_sched
|
|
||||||
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
|
|
||||||
# Mismatch: reachable & banner read, but not equal to target_version
|
|
||||||
- name: Build mismatch journal payload
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_mismatch_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
|
||||||
|
|
||||||
- name: Publish mismatch journal to control queue
|
|
||||||
when: journal_mismatch_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_mismatch_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_mismatch
|
|
||||||
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# SSH error path: TCP OK, but SSH failed
|
|
||||||
- name: Build failure journal payload (ssh error) + mark retry
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
{{ (banner_probe.stderr | default('') | trim) }}
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (ssh error) to control queue
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_fail
|
|
||||||
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
|
|
||||||
|
|
||||||
- name: Compute next-attempt delay (ms) according to policy
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
next_attempt: "{{ attempt | int + 1 }}"
|
|
||||||
next_delay_sec: >-
|
|
||||||
{% if attempt | int == 1 %}
|
|
||||||
300
|
|
||||||
{% elif attempt | int == 2 %}
|
|
||||||
600
|
|
||||||
{% else %}
|
|
||||||
0
|
|
||||||
{% endif %}
|
|
||||||
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
|
|
||||||
|
|
||||||
# If we've reached the cap, send a final “gave up” journal and stop.
|
|
||||||
- name: Build final gave-up journal (max attempts reached)
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_gaveup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
|
|
||||||
- name: Publish final gave-up journal
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_gaveup
|
|
||||||
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after final gave-up
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
# Otherwise schedule the next attempt (only if we still have budget)
|
|
||||||
- name: Build delayed after-upgrade payload for next attempt
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
delayed_payload:
|
|
||||||
task_name: "afterupgrade_check"
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
attempt: "{{ next_attempt | int }}"
|
|
||||||
max_attempts: "{{ effective_max_attempts | int }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at }}"
|
|
||||||
target_version: "{{ target_version_full }}"
|
|
||||||
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
||||||
schema_version: 1
|
|
||||||
|
|
||||||
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
|
|
||||||
when: delayed_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
routing_key: "deviceconfig"
|
|
||||||
payload: "{{ delayed_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_next
|
|
||||||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
@@ -1,359 +0,0 @@
|
|||||||
---
|
|
||||||
- name: After-upgrade verification (banner check + reporting)
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
# RabbitMQ + defaults (match the big script)
|
|
||||||
vars:
|
|
||||||
rmq_host: "10.210.12.2"
|
|
||||||
rmq_port: 15672
|
|
||||||
rmq_user: "admin"
|
|
||||||
rmq_pass: "change_me"
|
|
||||||
rmq_vhost: "app"
|
|
||||||
rmq_exchange: "controls"
|
|
||||||
control_queue: "queue_controls"
|
|
||||||
|
|
||||||
# Probing/SSH defaults
|
|
||||||
tcp_port: 22
|
|
||||||
nc_timeout: 5
|
|
||||||
ssh_user: "{{ ansible_user | default('root') }}"
|
|
||||||
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
|
|
||||||
ssh_timeout: 10
|
|
||||||
|
|
||||||
# Do NOT self-reference max_attempts. We’ll normalize below.
|
|
||||||
max_attempts_default: 3
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# ---- Normalize metadata safely (no self-referential defaults) ----
|
|
||||||
- name: Normalize metadata (no clever transforms)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
attempt: "{{ (attempt | default(1)) | int }}"
|
|
||||||
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
||||||
target_version: "{{ target_version | default('') }}"
|
|
||||||
# preserve the original string verbatim for all subsequent retries
|
|
||||||
target_version_full: "{{ target_version | default('') }}"
|
|
||||||
|
|
||||||
- name: Show received metadata
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "attempt={{ attempt }}"
|
|
||||||
- "max_attempts={{ effective_max_attempts }}"
|
|
||||||
- "correlation_id={{ correlation_id }}"
|
|
||||||
- "original_emitted_at={{ original_emitted_at }}"
|
|
||||||
- "target_version(full)={{ target_version_full }}"
|
|
||||||
|
|
||||||
# ---- Fast TCP reachability probe (controller-side) ----
|
|
||||||
- name: Check if TCP/{{ tcp_port }} is reachable with nc
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
|
|
||||||
register: nc_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Build failure journal (no TCP connectivity) + mark retry
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
|
|
||||||
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (no TCP connectivity)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_tcp_fail
|
|
||||||
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
|
|
||||||
- name: Stop host after TCP failure (we’ll schedule or close out below)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
ansible.builtin.meta: noop
|
|
||||||
|
|
||||||
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
|
|
||||||
- name: Probe banner via SSH from controller (classic extraction)
|
|
||||||
when: nc_probe.rc == 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
USER="{{ ssh_user }}"
|
|
||||||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
sshpass -p '{{ ssh_pass }}' \
|
|
||||||
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
|
|
||||||
"${USER}@${HOST}" \
|
|
||||||
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
|
|
||||||
register: banner_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Show the current version (banner line)
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "{{ banner_probe.stdout | trim }}"
|
|
||||||
|
|
||||||
- name: Evaluate version match (full-string contains check)
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
|
|
||||||
|
|
||||||
# ---- Journaling paths ----
|
|
||||||
|
|
||||||
# Success: banner matches expected full target_version
|
|
||||||
- name: Build success journal payload
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_success_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Banner='{{ (banner_probe.stdout | default('') | trim) }}'
|
|
||||||
Target='{{ target_version_full }}'
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
|
||||||
|
|
||||||
- name: Publish success journal to control queue
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_success_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_success
|
|
||||||
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# NEW: send a control tag to clean up device state on success
|
|
||||||
- name: Build cleanup control payload (update_cleanup_success)
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
control_cleanup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "update_cleanup_success"
|
|
||||||
|
|
||||||
- name: Publish cleanup control message to control queue
|
|
||||||
when: control_cleanup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ control_cleanup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_cleanup_success
|
|
||||||
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# Mismatch: reachable & banner read, but not equal to target_version
|
|
||||||
- name: Build mismatch journal payload
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_mismatch_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
|
|
||||||
|
|
||||||
- name: Publish mismatch journal to control queue
|
|
||||||
when: journal_mismatch_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_mismatch_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_mismatch
|
|
||||||
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# SSH error path: TCP OK, but SSH failed
|
|
||||||
- name: Build failure journal payload (ssh error) + mark retry
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
{{ (banner_probe.stderr | default('') | trim) }}
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (ssh error) to control queue
|
|
||||||
when: nc_probe.rc == 0 and banner_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_fail
|
|
||||||
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
|
|
||||||
|
|
||||||
- name: Compute next-attempt delay (ms) according to policy
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
next_attempt: "{{ attempt | int + 1 }}"
|
|
||||||
next_delay_sec: >-
|
|
||||||
{% if attempt | int == 1 %}
|
|
||||||
300
|
|
||||||
{% elif attempt | int == 2 %}
|
|
||||||
600
|
|
||||||
{% else %}
|
|
||||||
0
|
|
||||||
{% endif %}
|
|
||||||
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
|
|
||||||
|
|
||||||
# If we've reached the cap, send a final “gave up” journal and stop.
|
|
||||||
- name: Build final gave-up journal (max attempts reached)
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_gaveup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
|
|
||||||
- name: Publish final gave-up journal
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_gaveup
|
|
||||||
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after final gave-up
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
# Otherwise schedule the next attempt (only if we still have budget)
|
|
||||||
- name: Build delayed after-upgrade payload for next attempt
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
delayed_payload:
|
|
||||||
task_name: "afterupgrade_check"
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
attempt: "{{ next_attempt | int }}"
|
|
||||||
max_attempts: "{{ effective_max_attempts | int }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at }}"
|
|
||||||
target_version: "{{ target_version_full }}"
|
|
||||||
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
||||||
schema_version: 1
|
|
||||||
|
|
||||||
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
|
|
||||||
when: delayed_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
routing_key: "deviceconfig"
|
|
||||||
payload: "{{ delayed_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_next
|
|
||||||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
@@ -30,6 +30,8 @@
|
|||||||
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
||||||
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
||||||
|
|
||||||
|
debug_aic: false
|
||||||
|
|
||||||
tasks:
|
tasks:
|
||||||
# ---- Normalize metadata from delayed message ----
|
# ---- Normalize metadata from delayed message ----
|
||||||
- name: Normalize after upgrade metadata into facts
|
- name: Normalize after upgrade metadata into facts
|
||||||
@@ -353,13 +355,12 @@
|
|||||||
|
|
||||||
- name: Debug | banner normalization inputs
|
- name: Debug | banner normalization inputs
|
||||||
delegate_to: localhost
|
delegate_to: localhost
|
||||||
when: debug_aic | bool
|
when: (debug_aic | default(false)) | bool
|
||||||
ansible.builtin.debug:
|
ansible.builtin.debug:
|
||||||
msg:
|
msg:
|
||||||
- "banner_raw={{ (firmware_banner.stdout | default(firmware_banner) | default('')) | trim }}"
|
- "banner_raw={{ (dev2_fwver.stdout | default('') | trim) }}"
|
||||||
- "normalize rule: '2.2.1 rev 6801' -> '2.2.1-r6801'"
|
- "normalize rule: '2.2.1 rev 6801' -> '2.2.1-r6801'"
|
||||||
|
|
||||||
|
|
||||||
- name: Debug show raw firmware readout from DEV2
|
- name: Debug show raw firmware readout from DEV2
|
||||||
delegate_to: localhost
|
delegate_to: localhost
|
||||||
ansible.builtin.debug:
|
ansible.builtin.debug:
|
||||||
|
|||||||
@@ -1,201 +0,0 @@
|
|||||||
---
|
|
||||||
- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
vars:
|
|
||||||
# ---------------- BusyBox-safe path prefix ----------------
|
|
||||||
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
|
||||||
|
|
||||||
# ---------------- DEV1 (outer device) ----------------
|
|
||||||
dev1_user: "root"
|
|
||||||
dev1_pass: "wavewave"
|
|
||||||
dev1_iface: "br-wan"
|
|
||||||
|
|
||||||
# ---------------- DEV2 (indoor behind DEV1) ----------------
|
|
||||||
dev2_host: "192.168.1.1"
|
|
||||||
dev2_port: 22
|
|
||||||
dev2_side_ip: "192.168.1.11/24"
|
|
||||||
dev2_ssh_user: "root"
|
|
||||||
dev2_passfiles:
|
|
||||||
- "basicpass"
|
|
||||||
- "basicpass2"
|
|
||||||
|
|
||||||
# ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
|
|
||||||
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
||||||
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
||||||
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
||||||
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
||||||
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
||||||
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
||||||
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# ---------------- Step 1: Read DEV1 hostname and sanity ----------------
|
|
||||||
- name: Read DEV1 hostname
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
|
|
||||||
register: dev1_host_read
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if DEV1 hostname mismatch
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: (dev1_host_read.stdout | trim | length > 0) and
|
|
||||||
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
|
|
||||||
|
|
||||||
# ---------------- Step 2: Setup temporary IP for reachability ----------------
|
|
||||||
- name: Add temporary IP on DEV1 (ignore if exists)
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
||||||
register: add_ip
|
|
||||||
changed_when: add_ip.rc == 0
|
|
||||||
failed_when: >
|
|
||||||
add_ip.rc != 0 and
|
|
||||||
('File exists' not in (add_ip.stdout | default(''))) and
|
|
||||||
('File exists' not in (add_ip.stderr | default('')))
|
|
||||||
|
|
||||||
# ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
|
|
||||||
- name: Pick random free local port
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
for i in $(seq 1 25); do
|
|
||||||
p="$(shuf -i 20000-39999 -n1)"
|
|
||||||
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
|
|
||||||
done
|
|
||||||
exit 1
|
|
||||||
register: pick_port
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if no free local port
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: (pick_port.stdout | trim | length) == 0
|
|
||||||
|
|
||||||
- name: Record chosen local port and control dir
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_local_port: "{{ pick_port.stdout | trim }}"
|
|
||||||
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
|
|
||||||
|
|
||||||
- name: Build tunnel control socket path
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
|
|
||||||
|
|
||||||
|
|
||||||
- name: Start SSH ControlMaster tunnel via DEV1
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
sshpass -p '{{ dev1_pass }}' ssh -f -N \
|
|
||||||
-M -S "{{ _ctrl_sock }}" \
|
|
||||||
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
|
|
||||||
"{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: start_tunnel
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
# ---------------- Step 4: Determine working password for DEV2 ----------------
|
|
||||||
- name: Try both passfiles for DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
for f in {{ dev2_passfiles | join(' ') }}; do
|
|
||||||
if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
|
|
||||||
echo "$f"; exit 0;
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo NONE
|
|
||||||
register: dev2_passfile_try
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Save selected DEV2 passfile
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if no valid passfile found
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: dev2_passfile_used == "NONE"
|
|
||||||
|
|
||||||
# ---------------- Step 5: Read firmware version on DEV2 ----------------
|
|
||||||
- name: Read firmware version from DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
|
|
||||||
root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
|
|
||||||
register: dev2_fwver
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Show firmware version readout
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
# ---------------- Step 6: Publish result journal ----------------
|
|
||||||
- name: Build journal payload
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
|
|
||||||
tunnel_port={{ _local_port }}
|
|
||||||
|
|
||||||
- name: Publish journal to control queue
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_fwver
|
|
||||||
changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
post_tasks:
|
|
||||||
- name: Cleanup tunnel and temp IP
|
|
||||||
block:
|
|
||||||
- ansible.builtin.debug:
|
|
||||||
msg: "Cleaning up tunnel + temp IP"
|
|
||||||
changed_when: false
|
|
||||||
delegate_to: localhost
|
|
||||||
always:
|
|
||||||
- name: Close tunnel
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Remove control dir
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.file:
|
|
||||||
path: "{{ _ctrl_dir | default('/tmp/none') }}"
|
|
||||||
state: absent
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Remove temporary IP from DEV1
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
||||||
register: del_ip
|
|
||||||
failed_when: false
|
|
||||||
changed_when: false
|
|
||||||
@@ -1,644 +0,0 @@
|
|||||||
---
|
|
||||||
- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
vars:
|
|
||||||
# ---------------- BusyBox-safe path prefix ----------------
|
|
||||||
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
|
||||||
|
|
||||||
# ---------------- DEV1 (outer device) ----------------
|
|
||||||
dev1_user: "root"
|
|
||||||
dev1_pass: "wavewave"
|
|
||||||
dev1_iface: "br-wan"
|
|
||||||
|
|
||||||
# ---------------- DEV2 (indoor behind DEV1) ----------------
|
|
||||||
dev2_host: "192.168.1.1"
|
|
||||||
dev2_port: 22
|
|
||||||
dev2_side_ip: "192.168.1.11/24"
|
|
||||||
dev2_ssh_user: "root"
|
|
||||||
dev2_passfiles:
|
|
||||||
- "basicpass"
|
|
||||||
- "basicpass2"
|
|
||||||
|
|
||||||
# ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
|
|
||||||
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
||||||
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
||||||
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
||||||
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
||||||
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
||||||
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
||||||
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# ---- Normalize metadata (from delayed message) ----
|
|
||||||
- name: Normalize after-upgrade metadata
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
attempt: "{{ (attempt | default(1)) | int }}"
|
|
||||||
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
||||||
target_version: "{{ target_version | default('') }}"
|
|
||||||
target_version_full: "{{ target_version | default('') }}"
|
|
||||||
|
|
||||||
# ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
|
|
||||||
- name: Check if TCP/22 on DEV1 is reachable
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
|
|
||||||
register: nc_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
# ---- If TCP down: journal + schedule next try or give up ----
|
|
||||||
- name: Build failure journal (TCP unreachable) + mark retry
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
TCP 22 unreachable. Correlation={{ correlation_id }}
|
|
||||||
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
_needs_retry: true
|
|
||||||
|
|
||||||
- name: Publish failure journal (TCP unreachable)
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers: { content-type: "application/json" }
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties: { content_type: "application/json" }
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_tcp_fail
|
|
||||||
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Compute next-attempt delay (10 minutes) and counters
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
next_attempt: "{{ attempt | int + 1 }}"
|
|
||||||
next_delay_sec: 600
|
|
||||||
next_delay_ms: 600000
|
|
||||||
|
|
||||||
- name: Build final gave-up journal (max attempts reached)
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_gaveup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Exhausted attempts. Last error path=TCP.
|
|
||||||
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
|
|
||||||
|
|
||||||
- name: Publish final gave-up journal
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers: { content-type: "application/json" }
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties: { content_type: "application/json" }
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_j_gaveup
|
|
||||||
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after final gave-up
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
- name: Build delayed after-upgrade payload for next attempt (10m)
|
|
||||||
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
delayed_payload:
|
|
||||||
task_name: "afterupgrade_indoor_check"
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
attempt: "{{ next_attempt | int }}"
|
|
||||||
max_attempts: "{{ effective_max_attempts | int }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at }}"
|
|
||||||
target_version: "{{ target_version_full }}"
|
|
||||||
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
||||||
schema_version: 1
|
|
||||||
|
|
||||||
- name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
|
|
||||||
when: delayed_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers: { content-type: "application/json" }
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
||||||
correlation_id: "{{ correlation_id }}"
|
|
||||||
routing_key: "deviceconfig"
|
|
||||||
payload: "{{ delayed_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_next
|
|
||||||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Stop host after scheduling next attempt
|
|
||||||
when: (_needs_retry | default(false)) | bool
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------- Step 1: Read DEV1 hostname and sanity ----------------
|
|
||||||
- name: Read DEV1 hostname
|
|
||||||
when: nc_probe.rc == 0
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
|
|
||||||
register: dev1_host_read
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if DEV1 hostname mismatch
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: (dev1_host_read.stdout | trim | length > 0) and
|
|
||||||
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
|
|
||||||
|
|
||||||
# ---------------- Step 2: Setup temporary IP for reachability ----------------
|
|
||||||
- name: Add temporary IP on DEV1 (ignore if exists)
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
||||||
register: add_ip
|
|
||||||
changed_when: add_ip.rc == 0
|
|
||||||
failed_when: >
|
|
||||||
add_ip.rc != 0 and
|
|
||||||
('File exists' not in (add_ip.stdout | default(''))) and
|
|
||||||
('File exists' not in (add_ip.stderr | default('')))
|
|
||||||
|
|
||||||
# ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
|
|
||||||
- name: Pick random free local port
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
for i in $(seq 1 25); do
|
|
||||||
p="$(shuf -i 20000-39999 -n1)"
|
|
||||||
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
|
|
||||||
done
|
|
||||||
exit 1
|
|
||||||
register: pick_port
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if no free local port
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: (pick_port.stdout | trim | length) == 0
|
|
||||||
|
|
||||||
- name: Record chosen local port and control dir
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_local_port: "{{ pick_port.stdout | trim }}"
|
|
||||||
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
|
|
||||||
|
|
||||||
- name: Build tunnel control socket path
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
|
|
||||||
|
|
||||||
- name: Start SSH ControlMaster tunnel via DEV1
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
sshpass -p '{{ dev1_pass }}' ssh -f -N \
|
|
||||||
-M -S "{{ _ctrl_sock }}" \
|
|
||||||
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
|
|
||||||
"{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: start_tunnel
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
# --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
|
|
||||||
- name: Small delay for tunnel to settle
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.wait_for:
|
|
||||||
timeout: 1
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Verify tunnel master running (ssh -O check)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
|
|
||||||
register: tun_check
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Debug tunnel check
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "tunnel_check.rc={{ tun_check.rc }}"
|
|
||||||
- "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
- name: Sanity confirm tunnel TCP reachability to DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
nc -z -w5 127.0.0.1 "{{ _local_port }}"
|
|
||||||
register: nc_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Debug reachability result
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "nc.rc={{ nc_probe.rc }}"
|
|
||||||
- "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
|
|
||||||
- "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
|
|
||||||
|
|
||||||
- name: Stop if tunnel TCP check failed
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
|
|
||||||
# ---------------- Step 4: Determine working password for DEV2 ----------------
|
|
||||||
- name: Try both passfiles for DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
for f in {{ dev2_passfiles | join(' ') }}; do
|
|
||||||
if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
|
|
||||||
echo "$f"; exit 0;
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo NONE
|
|
||||||
register: dev2_passfile_try
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Save selected DEV2 passfile
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Debug selected DEV2 passfile
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "dev2_passfile_used={{ dev2_passfile_used }}"
|
|
||||||
|
|
||||||
- name: Stop if no valid passfile found
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: dev2_passfile_used == "NONE"
|
|
||||||
|
|
||||||
# ---------------- Step 5: Read firmware version on DEV2 ----------------
|
|
||||||
- name: Read firmware version from DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
|
|
||||||
root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
|
|
||||||
register: dev2_fwver
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Show firmware version readout
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
# ---------------- Retry metadata + success evaluation ----------------
|
|
||||||
- name: Normalize retry metadata for indoor checker
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
attempt: "{{ (attempt | default(1)) | int }}"
|
|
||||||
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
||||||
|
|
||||||
- name: Evaluate indoor success (firmware read OK)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
indoor_success: "{{ (dev2_fwver.rc | default(1) == 0)
|
|
||||||
and ((dev2_fwver.stdout | default('') | trim) | length > 0)
|
|
||||||
and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
|
|
||||||
|
|
||||||
# ---------------- Step 6: Publish result journal (+ tags on success) ----------------
|
|
||||||
# SUCCESS PATH: add tag, remove scheduled tag, and journal with "success"
|
|
||||||
- name: Build success journal payload
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_success_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
|
|
||||||
tunnel_port={{ _local_port }}
|
|
||||||
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
||||||
|
|
||||||
- name: Publish success journal to control queue
|
|
||||||
when: journal_success_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_success_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_success
|
|
||||||
changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Build payload to add indoor-update-success tag (DEV1)
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
tag_add_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "tag_add"
|
|
||||||
task_result: "indoor-update-success"
|
|
||||||
|
|
||||||
- name: Publish indoor-update-success tag
|
|
||||||
when: tag_add_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ tag_add_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_tag_add_success
|
|
||||||
changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Build payload to remove indoor-restart-scheduled tag (DEV1)
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
tag_remove_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "tag_remove"
|
|
||||||
task_result: "indoor-restart-scheduled"
|
|
||||||
|
|
||||||
- name: Publish indoor-restart-scheduled tag removal
|
|
||||||
when: tag_remove_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ tag_remove_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_tag_remove_sched
|
|
||||||
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# --- Normalize firmware string and set custom field on success ---
|
|
||||||
- name: Capture raw firmware banner from DEV2 (for normalization)
|
|
||||||
when: indoor_success | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
- name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
|
|
||||||
when:
|
|
||||||
- indoor_success | bool
|
|
||||||
- (fw_banner_raw | default('') | length) > 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
fw_norm: >-
|
|
||||||
{{
|
|
||||||
fw_banner_raw
|
|
||||||
if (fw_banner_raw | lower is search('-r[0-9]+$'))
|
|
||||||
else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
|
|
||||||
}}
|
|
||||||
|
|
||||||
- name: Debug normalized firmware (indoor_fwver)
|
|
||||||
when: fw_norm is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
|
|
||||||
|
|
||||||
- name: Publish custom_field_set indoor_fwver
|
|
||||||
when:
|
|
||||||
- indoor_success | bool
|
|
||||||
- fw_norm is defined
|
|
||||||
- (fw_norm | length) > 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ {
|
|
||||||
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
||||||
'task_name': 'custom_field_set',
|
|
||||||
'task_add1': 'indoor_fwver',
|
|
||||||
'task_result': fw_norm
|
|
||||||
} | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_customfield_fw
|
|
||||||
changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
|
|
||||||
- name: Build failure journal payload (indoor firmware read failed)
|
|
||||||
when: not (indoor_success | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_fail_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
|
|
||||||
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
||||||
|
|
||||||
- name: Publish failure journal to control queue
|
|
||||||
when: journal_fail_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_fail_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_fail
|
|
||||||
changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
- name: Compute retry parameters (10 minutes)
|
|
||||||
when: not (indoor_success | bool)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
next_attempt: "{{ (attempt | int) + 1 }}"
|
|
||||||
next_delay_sec: 600
|
|
||||||
next_delay_ms: "{{ 600000 }}"
|
|
||||||
|
|
||||||
- name: Build final gave-up journal (max attempts reached)
|
|
||||||
when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_gaveup_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
|
|
||||||
Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
|
|
||||||
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
|
|
||||||
|
|
||||||
- name: Publish final gave-up journal
|
|
||||||
when: journal_gaveup_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_gaveup_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_gaveup
|
|
||||||
changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# Only schedule next attempt if we still have budget left
|
|
||||||
- name: Build delayed payload for next indoor attempt (10 min)
|
|
||||||
when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
delayed_payload:
|
|
||||||
task_name: "afterupgrade_indoor_check"
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
attempt: "{{ next_attempt | int }}"
|
|
||||||
max_attempts: "{{ effective_max_attempts | int }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
original_emitted_at: "{{ original_emitted_at | default('') }}"
|
|
||||||
current_delay_sec: "{{ next_delay_sec | int }}"
|
|
||||||
schema_version: 1
|
|
||||||
|
|
||||||
- name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
|
|
||||||
when: delayed_payload is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
expiration: "{{ (next_delay_ms | int) | string }}"
|
|
||||||
correlation_id: "{{ correlation_id | default('') }}"
|
|
||||||
routing_key: "deviceconfig"
|
|
||||||
payload: "{{ delayed_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_pub_next
|
|
||||||
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
post_tasks:
|
|
||||||
- name: Cleanup tunnel and temp IP
|
|
||||||
block:
|
|
||||||
- ansible.builtin.debug:
|
|
||||||
msg: "Cleaning up tunnel + temp IP"
|
|
||||||
changed_when: false
|
|
||||||
delegate_to: localhost
|
|
||||||
always:
|
|
||||||
- name: Close tunnel
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Remove control dir
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.file:
|
|
||||||
path: "{{ _ctrl_dir | default('/tmp/none') }}"
|
|
||||||
state: absent
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Remove temporary IP from DEV1
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
||||||
register: del_ip
|
|
||||||
failed_when: false
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
@@ -1,997 +0,0 @@
|
|||||||
---
|
|
||||||
# update-secondline.yml
|
|
||||||
- name: Second-line upgrade tunneled noninvasive engine
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
vars:
|
|
||||||
# Busybox-safe PATH prefix for all remote raw calls on DEV1
|
|
||||||
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
|
||||||
|
|
||||||
# DEV1 credentials (stable, like rebootin222)
|
|
||||||
dev1_user: "root"
|
|
||||||
dev1_pass: "wavewave"
|
|
||||||
|
|
||||||
# Tunnel target DEV2 behind DEV1
|
|
||||||
dev2_host: "192.168.1.1"
|
|
||||||
dev2_port: 22
|
|
||||||
|
|
||||||
# Temp IP we add to DEV1 so it can reach DEV2
|
|
||||||
dev2_side_ip: "192.168.1.11/24"
|
|
||||||
dev1_iface: "br-wan"
|
|
||||||
|
|
||||||
# DEV2 behind the tunnel
|
|
||||||
dev2_ssh_user: "root"
|
|
||||||
dev2_passfiles:
|
|
||||||
- "basicpass"
|
|
||||||
- "basicpass2"
|
|
||||||
|
|
||||||
# SSH options used from controller
|
|
||||||
ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
|
|
||||||
|
|
||||||
# Image to stage on DEV2 noninvasive we do not run update -w here
|
|
||||||
image_filename: "fox200-2.2.1-r6801.bin"
|
|
||||||
image_md5: "56b7211709de617e058b98d4204e2562"
|
|
||||||
# Optional SHA256; leave empty to skip SHA256 checks
|
|
||||||
image_sha256: ""
|
|
||||||
dev2_image_dir: "/tmp"
|
|
||||||
dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
|
|
||||||
|
|
||||||
# Reboot delay in minutes integer string if set we will schedule reboot on DEV2
|
|
||||||
rebootin: ""
|
|
||||||
|
|
||||||
# ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
|
|
||||||
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
||||||
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
||||||
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
||||||
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
||||||
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
||||||
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
||||||
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
|
||||||
|
|
||||||
pre_tasks:
|
|
||||||
# ------------------------------- Hostname sanity DEV1 -------------------------------
|
|
||||||
- name: Read DEV1 hostname from env busybox safe
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
|
|
||||||
register: dev1_host_read
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if DEV1 connected hostname differs from inventory
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: (dev1_host_read.stdout | trim | length > 0) and
|
|
||||||
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# ---------------------------- Idempotent temp IP on DEV1 ----------------------------
|
|
||||||
- name: Add temporary IP on DEV1 idempotent treat File exists as OK
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
||||||
register: add_ip
|
|
||||||
changed_when: add_ip.rc == 0
|
|
||||||
failed_when: >
|
|
||||||
add_ip.rc != 0
|
|
||||||
and ('File exists' not in (add_ip.stdout | default('')))
|
|
||||||
and ('File exists' not in (add_ip.stderr | default('')))
|
|
||||||
|
|
||||||
- name: Debug add ip results
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "add_ip.rc={{ add_ip.rc | default('') }}"
|
|
||||||
- "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
|
|
||||||
- "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
|
|
||||||
|
|
||||||
# ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ----------------------------
|
|
||||||
- name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
|
|
||||||
register: dev2_mac_scan
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Set dev2_mac from fdb scan (if any)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Remove current ARP entry for DEV2 on DEV1 (best-effort)
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
|
|
||||||
register: dev2_arp_del
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
- name: Add static ARP for DEV2 on DEV1
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
|
|
||||||
register: dev2_arp_add
|
|
||||||
changed_when: dev2_arp_add.rc == 0
|
|
||||||
failed_when: >
|
|
||||||
dev2_arp_add.rc != 0
|
|
||||||
and ('File exists' not in (dev2_arp_add.stdout | default('')))
|
|
||||||
and ('File exists' not in (dev2_arp_add.stderr | default('')))
|
|
||||||
|
|
||||||
- name: Debug static ARP result
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "dev2_mac={{ dev2_mac | default('UNSET') }}"
|
|
||||||
- "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
|
|
||||||
- "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
|
|
||||||
- "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
|
|
||||||
|
|
||||||
- name: Skip ARP add because MAC not found
|
|
||||||
when: dev2_mac is not defined or dev2_mac | length == 0
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
|
|
||||||
|
|
||||||
# ---------------------------- Local tunnel preparation ----------------------------
|
|
||||||
- name: Pick a free local port for the tunnel controller
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
pick() {
|
|
||||||
for i in $(seq 1 25); do
|
|
||||||
p="$(shuf -i 20000-39999 -n 1)"
|
|
||||||
if command -v ss >/dev/null 2>&1; then
|
|
||||||
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
|
|
||||||
echo "$p"; return 0
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
|
|
||||||
echo "$p"; return 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
pick
|
|
||||||
register: pick_port
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Stop if no free local port
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: (pick_port.stdout | trim | length) == 0
|
|
||||||
|
|
||||||
- name: Record chosen local port and control directory
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_local_port: "{{ pick_port.stdout | trim }}"
|
|
||||||
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
|
|
||||||
|
|
||||||
- name: Build tunnel control socket path
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
|
|
||||||
|
|
||||||
- name: Debug chosen tunnel params
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "local_port={{ _local_port }}"
|
|
||||||
- "ctrl_dir={{ _ctrl_dir }}"
|
|
||||||
- "ctrl_sock={{ _ctrl_sock }}"
|
|
||||||
|
|
||||||
- name: issue arping with own IP
|
|
||||||
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
|
|
||||||
|
|
||||||
# ---------------------------- Start SSH local forward via DEV1 ----------------------------
|
|
||||||
- name: Start SSH tunnel via DEV1 background ControlMaster
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
USER="{{ dev1_user }}"
|
|
||||||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
|
|
||||||
-M -S "{{ _ctrl_sock }}" \
|
|
||||||
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
|
|
||||||
"${USER}@${HOST}"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: start_tunnel
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Small delay for tunnel to settle
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.wait_for:
|
|
||||||
timeout: 1
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Verify tunnel master running proper O check with destination
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
||||||
ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
|
|
||||||
register: tun_check
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Debug tunnel check
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "tunnel_check.rc={{ tun_check.rc }}"
|
|
||||||
- "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
# ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
|
|
||||||
- name: Sanity confirm tunnel TCP reachability to DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
nc -z -w5 127.0.0.1 "{{ _local_port }}"
|
|
||||||
register: nc_probe
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Debug reachability result
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "nc.rc={{ nc_probe.rc }}"
|
|
||||||
- "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
|
|
||||||
- "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
|
|
||||||
|
|
||||||
- name: Stop if tunnel TCP check failed
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
when: nc_probe.rc != 0
|
|
||||||
|
|
||||||
- name: Sanity show presence and permissions of passfiles on controller
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
|
|
||||||
register: dev2_ls
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Debug passfiles listing
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "{{ (dev2_ls.stdout | default('')) | trim }}"
|
|
||||||
- "{{ (dev2_ls.stderr | default('')) | trim }}"
|
|
||||||
|
|
||||||
# ---------------------------- Pick DEV2 password root only ----------------------------
|
|
||||||
- name: Try DEV2 login with basicpass root
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f basicpass ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
|
|
||||||
register: dev2_try_basicpass
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Select basicpass as working passfile
|
|
||||||
when: dev2_try_basicpass.rc == 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_passfile_used: "basicpass"
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Try DEV2 login with basicpass2 root only if first failed
|
|
||||||
when: dev2_passfile_used is not defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f basicpass2 ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
|
|
||||||
register: dev2_try_basicpass2
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Select basicpass2 as working passfile
|
|
||||||
when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_passfile_used: "basicpass2"
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Set dev2_passfile_used to NONE if neither worked
|
|
||||||
when: dev2_passfile_used is not defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_passfile_used: "NONE"
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Debug selected DEV2 passfile or NONE
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "dev2_passfile_used={{ dev2_passfile_used }}"
|
|
||||||
- "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}"
|
|
||||||
- "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}"
|
|
||||||
|
|
||||||
- name: Read DEV2 hostname via tunnel (busybox-safe)
|
|
||||||
when: dev2_passfile_used != "NONE"
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_host_read
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Normalize hostnames for strict compare
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
|
|
||||||
_dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
|
|
||||||
_dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
|
|
||||||
|
|
||||||
- name: Debug hostname bytes (hex)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "inv={{ _inv_hn | tojson }}"
|
|
||||||
- "dev1={{ _dev1_hn | tojson }}"
|
|
||||||
- "dev2={{ _dev2_hn | tojson }}"
|
|
||||||
|
|
||||||
# ---------------- Hostname equality guard (soft-journal and stop) ----------------
|
|
||||||
- name: Enforce DEV2 hostname equals inventory and DEV1
|
|
||||||
block:
|
|
||||||
- name: Fail if DEV2 hostname differs from inventory/DEV1
|
|
||||||
ansible.builtin.fail:
|
|
||||||
msg: >
|
|
||||||
Hostname mismatch: DEV2='{{ _dev2_hn }}',
|
|
||||||
inventory='{{ _inv_hn }}',
|
|
||||||
DEV1='{{ _dev1_hn }}'
|
|
||||||
when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
|
|
||||||
rescue:
|
|
||||||
- name: Initialize journal array for hostname mismatch
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_journal: []
|
|
||||||
_blocked: true
|
|
||||||
_prep_blocked: false
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Append hostname mismatch to journal
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}"
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Build control queue payload for indoor aborted journal (hostname)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_indoor_aborted:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Publish indoor aborted journal to control queue (hostname)
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_indoor_aborted | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_journal_indoor_aborted_hn_resp
|
|
||||||
changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
|
|
||||||
failed_when: >
|
|
||||||
(rmq_journal_indoor_aborted_hn_resp.status != 200) or
|
|
||||||
(rmq_journal_indoor_aborted_hn_resp.json is not defined) or
|
|
||||||
(not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Stop host after hostname mismatch
|
|
||||||
ansible.builtin.meta: end_host
|
|
||||||
|
|
||||||
# ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
|
|
||||||
- name: Initialize journal and flags
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_journal: []
|
|
||||||
_prep_blocked: false
|
|
||||||
_blocked: false
|
|
||||||
|
|
||||||
- name: Build specific marker filename (no .bin)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}"
|
|
||||||
|
|
||||||
- name: Check for any preparation markers on DEV2 (wildcard count)
|
|
||||||
when: dev2_passfile_used != "NONE"
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"ls /tmp/prepared_for* 2>/dev/null | wc -l"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_prep_count
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Soft-block if any prep markers already present
|
|
||||||
when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_prep_blocked: true
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
|
|
||||||
|
|
||||||
# ---------------------------- DEV2 version firmux primary check ----------------------------
|
|
||||||
- name: Read DEV2 firmux release
|
|
||||||
when: dev2_passfile_used != "NONE"
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"cat /usr/lib/release/firmux 2>/dev/null || true"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_firmux
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Debug firmux content DEV2
|
|
||||||
when: dev2_firmux is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
# ---------------------------- Normalize rebootin early ----------------------------
|
|
||||||
- name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)"
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_reboot_raw: "{{ rebootin | default('') | string | trim | lower }}"
|
|
||||||
_reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}"
|
|
||||||
_reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}"
|
|
||||||
_reboot_minutes: >-
|
|
||||||
{{
|
|
||||||
0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'])
|
|
||||||
else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1)
|
|
||||||
}}
|
|
||||||
|
|
||||||
- name: "Normalize rebootin (phase 2: requested flag)"
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}"
|
|
||||||
|
|
||||||
# ---------------------------- Journal: indoor start (after we know we can proceed) ----------------------------
|
|
||||||
- name: Build control queue payload for indoor start journal
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_indoor_start:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
Indoor: Dev2 is reachable, starting with update.
|
|
||||||
image={{ image_filename }}, md5={{ image_md5 }},
|
|
||||||
firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }},
|
|
||||||
reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }}
|
|
||||||
when: dev2_passfile_used != "NONE"
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Publish indoor start journal to control queue
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_indoor_start | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_journal_indoor_start_resp
|
|
||||||
changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
|
|
||||||
failed_when: >
|
|
||||||
(rmq_journal_indoor_start_resp.status != 200) or
|
|
||||||
(rmq_journal_indoor_start_resp.json is not defined) or
|
|
||||||
(not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
|
|
||||||
when: journal_indoor_start is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
# ---------------------------- Stage image on DEV2 no write ----------------------------
|
|
||||||
- name: Check local presence of image file
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.stat:
|
|
||||||
path: "{{ image_filename }}"
|
|
||||||
register: local_img
|
|
||||||
|
|
||||||
- name: Soft-block if image missing locally
|
|
||||||
when: not local_img.stat.exists
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
|
|
||||||
|
|
||||||
- name: Compute local md5 of the image
|
|
||||||
when: local_img.stat.exists
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
md5sum "{{ image_filename }}" | awk '{print $1}'
|
|
||||||
register: local_md5
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Verify local md5 matches expected
|
|
||||||
when: local_img.stat.exists
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.assert:
|
|
||||||
that:
|
|
||||||
- (local_md5.stdout | trim) == image_md5
|
|
||||||
fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
|
|
||||||
success_msg: "Local md5 matches expected"
|
|
||||||
|
|
||||||
# Optional SHA256 local
|
|
||||||
- name: Compute local sha256 of the image (if provided)
|
|
||||||
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
sha256sum "{{ image_filename }}" | awk '{print $1}'
|
|
||||||
register: local_sha256
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Soft-block if local sha256 mismatch or unavailable
|
|
||||||
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_prep_blocked: true
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
|
|
||||||
|
|
||||||
# fw_printenv health before upload (soft-fail)
|
|
||||||
- name: Read fw_printenv size (line count) on DEV2
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"(fw_printenv 2>/dev/null | wc -l) || echo 0"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_fwenv_wc
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Soft-block if fw_printenv too small
|
|
||||||
when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_prep_blocked: true
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
|
|
||||||
|
|
||||||
- name: Check existing image md5 on DEV2 NOFILE if absent
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_md5_before
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Copy image to DEV2 via tunnel only if missing or md5 mismatch
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" scp \
|
|
||||||
-P "$PORT" \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
"{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: scp_push
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Compute md5 of image on DEV2 after copy or if exists
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_md5_after
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Verify DEV2 md5 matches expected (soft-fail journal)
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_prep_blocked: true
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
|
|
||||||
|
|
||||||
# Optional SHA256 remote
|
|
||||||
- name: Compute sha256 of image on DEV2 (if provided)
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_sha256_after
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Soft-block if DEV2 sha256 mismatch/unavailable
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_prep_blocked: true
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
|
|
||||||
|
|
||||||
- name: Run noninvasive update check on DEV2 update c
|
|
||||||
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=15 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"update -c '{{ dev2_image_path }}' 2>&1 || true"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_update_check
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Debug update c output DEV2
|
|
||||||
when: dev2_update_check is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
|
|
||||||
|
|
||||||
- name: Soft-block if update -c did not say valid
|
|
||||||
when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_prep_blocked: true
|
|
||||||
_blocked: true
|
|
||||||
_journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
|
|
||||||
|
|
||||||
# ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
|
|
||||||
- name: Build control queue payload for indoor aborted journal
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_indoor_aborted:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
|
|
||||||
when: (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Publish indoor aborted journal to control queue
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_indoor_aborted | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_journal_indoor_aborted_resp
|
|
||||||
changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
|
|
||||||
failed_when: >
|
|
||||||
(rmq_journal_indoor_aborted_resp.status != 200) or
|
|
||||||
(rmq_journal_indoor_aborted_resp.json is not defined) or
|
|
||||||
(not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
|
|
||||||
when: journal_indoor_aborted is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
# ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
|
|
||||||
- name: Upgrade write and flip block
|
|
||||||
when: not (_blocked | default(false))
|
|
||||||
block:
|
|
||||||
- name: Write image on DEV2 (this will take a while)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=0 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"update -w '{{ dev2_image_path }}'"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_up_write
|
|
||||||
changed_when: true
|
|
||||||
failed_when: dev2_up_write.stdout is not search('update is complete')
|
|
||||||
|
|
||||||
- name: Read current active partition on DEV2
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"fw_printenv active | awk -F= '/^active=/{print \$2}'"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_active_before
|
|
||||||
changed_when: false
|
|
||||||
failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
|
|
||||||
|
|
||||||
- name: Determine new active value for DEV2
|
|
||||||
when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
|
|
||||||
|
|
||||||
- name: Switch active partition to {{ dev2_new_active }}
|
|
||||||
when: dev2_new_active is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"fw_setenv active {{ dev2_new_active }}"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_setenv_out
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
- name: Verify active partition flipped on DEV2
|
|
||||||
when: dev2_new_active is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"fw_printenv active | awk -F= '/^active=/{print \$2}'"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_active_after
|
|
||||||
changed_when: false
|
|
||||||
failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
|
|
||||||
|
|
||||||
- name: Create specific prep marker on DEV2 (no generic)
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
||||||
"touch '{{ _marker_specific }}'"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_marker_write
|
|
||||||
changed_when: true
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
# ---------------------------- Reboot scheduling (normalized) ----------------------------
|
|
||||||
- name: Schedule delayed reboot on DEV2 minutes
|
|
||||||
when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
set -e
|
|
||||||
PORT="{{ _local_port }}"
|
|
||||||
SECS="$(( {{ _reboot_minutes | int }} * 60 ))"
|
|
||||||
CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
|
|
||||||
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
||||||
-o AddressFamily=inet \
|
|
||||||
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
||||||
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
|
|
||||||
-o ConnectTimeout=10 \
|
|
||||||
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: dev2_reboot_sched
|
|
||||||
changed_when: true
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Note reboot request bad format
|
|
||||||
when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"
|
|
||||||
|
|
||||||
# ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
|
|
||||||
- name: Build control queue payload for indoor updated journal
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
_write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}"
|
|
||||||
when: not (_blocked | default(false))
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Build indoor updated journal payload text
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
journal_indoor_updated:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
Indoor: Dev2 updated, scheduling reboot {{
|
|
||||||
'now' if (_reboot_requested and (_reboot_minutes | int) == 0)
|
|
||||||
else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0)
|
|
||||||
else 'not requested'
|
|
||||||
}}.
|
|
||||||
write_done={{ _write_success }},
|
|
||||||
active_before={{ (dev2_active_before.stdout | default('NA')) | trim }},
|
|
||||||
active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}
|
|
||||||
when: not (_blocked | default(false)) and (_write_success | bool)
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
- name: Publish indoor updated journal to control queue
|
|
||||||
ansible.builtin.uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ journal_indoor_updated | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_journal_indoor_updated_resp
|
|
||||||
changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
|
|
||||||
failed_when: >
|
|
||||||
(rmq_journal_indoor_updated_resp.status != 200) or
|
|
||||||
(rmq_journal_indoor_updated_resp.json is not defined) or
|
|
||||||
(not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
|
|
||||||
when: journal_indoor_updated is defined
|
|
||||||
delegate_to: localhost
|
|
||||||
|
|
||||||
# ---------------------------- summary ----------------------------
|
|
||||||
- name: Summary show key results
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "dev2_passfile_used={{ dev2_passfile_used }}"
|
|
||||||
- "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
|
|
||||||
- "local_image_present={{ local_img.stat.exists | default(false) }}"
|
|
||||||
- "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
|
|
||||||
- "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
|
|
||||||
- "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
|
|
||||||
- "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}"
|
|
||||||
- "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
|
|
||||||
- "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
|
|
||||||
- "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
|
|
||||||
- "reboot_requested={{ _reboot_requested | default(false) }}"
|
|
||||||
- "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}"
|
|
||||||
- "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}"
|
|
||||||
- "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
|
|
||||||
- "prep_blocked={{ _prep_blocked | default(false) }}"
|
|
||||||
- "blocked={{ _blocked | default(false) }}"
|
|
||||||
- "journal={{ (_journal | default([])) | join(' || ') }}"
|
|
||||||
|
|
||||||
post_tasks:
|
|
||||||
- name: Cleanup always
|
|
||||||
block:
|
|
||||||
- ansible.builtin.debug:
|
|
||||||
msg: "entering cleanup block"
|
|
||||||
changed_when: false
|
|
||||||
delegate_to: localhost
|
|
||||||
always:
|
|
||||||
- name: Close tunnel best effort
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.shell: |
|
|
||||||
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
|
|
||||||
changed_when: false
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Remove control dir best effort
|
|
||||||
delegate_to: localhost
|
|
||||||
ansible.builtin.file:
|
|
||||||
path: "{{ _ctrl_dir | default('/tmp/none') }}"
|
|
||||||
state: absent
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK
|
|
||||||
ansible.builtin.raw: >
|
|
||||||
{{ pathprefix }}
|
|
||||||
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
||||||
register: del_ip
|
|
||||||
changed_when: del_ip.rc == 0
|
|
||||||
failed_when: >
|
|
||||||
del_ip.rc != 0
|
|
||||||
and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
|
|
||||||
and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
|
|
||||||
|
|
||||||
- name: Debug del ip results
|
|
||||||
ansible.builtin.debug:
|
|
||||||
msg:
|
|
||||||
- "del_ip.rc={{ del_ip.rc | default('') }}"
|
|
||||||
- "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
|
|
||||||
- "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
|
|
||||||
when: del_ip is defined
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,441 +0,0 @@
|
|||||||
---
|
|
||||||
- name: Deploy WiFi debug (single device, linear)
|
|
||||||
hosts: all
|
|
||||||
gather_facts: no
|
|
||||||
|
|
||||||
vars:
|
|
||||||
remote_syslog_ip: "102.38.125.161"
|
|
||||||
|
|
||||||
# Program version (0)
|
|
||||||
wdbg_version: "v17"
|
|
||||||
|
|
||||||
ssh_user: "{{ ansible_user | default('root') }}"
|
|
||||||
ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
|
|
||||||
|
|
||||||
# RabbitMQ (use controls exchange + queue_controls like the reference) (e)
|
|
||||||
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
||||||
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
||||||
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
||||||
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
||||||
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
||||||
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
||||||
control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
|
|
||||||
- block:
|
|
||||||
|
|
||||||
# --- SSH reachability check ---
|
|
||||||
- name: Check SSH connectivity (raw ping)
|
|
||||||
raw: "echo ping"
|
|
||||||
register: ping_result
|
|
||||||
ignore_errors: true
|
|
||||||
|
|
||||||
- block:
|
|
||||||
|
|
||||||
############ step 1
|
|
||||||
|
|
||||||
- name: Check if crontab launch is already present in sysstart.lua
|
|
||||||
raw: |
|
|
||||||
grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
|
|
||||||
register: sysstart_cron_check
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
- name: Announce presence and skip edits
|
|
||||||
when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
|
|
||||||
debug:
|
|
||||||
msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
|
|
||||||
|
|
||||||
- name: Fetch sysstart.lua via scp (password auth)
|
|
||||||
when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
|
|
||||||
delegate_to: localhost
|
|
||||||
command: >
|
|
||||||
sshpass -p {{ ssh_pass | quote }}
|
|
||||||
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
||||||
{{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
|
|
||||||
/tmp/{{ inventory_hostname }}_sysstart.lua
|
|
||||||
register: scp_get
|
|
||||||
retries: 3
|
|
||||||
delay: 2
|
|
||||||
until: scp_get.rc == 0
|
|
||||||
|
|
||||||
- name: Edit the file locally
|
|
||||||
when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
|
|
||||||
delegate_to: localhost
|
|
||||||
lineinfile:
|
|
||||||
path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
|
|
||||||
line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
|
|
||||||
insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
|
|
||||||
|
|
||||||
- name: Push the file back (write to -2)
|
|
||||||
when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
|
|
||||||
delegate_to: localhost
|
|
||||||
command: >
|
|
||||||
sshpass -p {{ ssh_pass | quote }}
|
|
||||||
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
||||||
/tmp/{{ inventory_hostname }}_sysstart.lua
|
|
||||||
{{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
|
|
||||||
register: scp_put
|
|
||||||
retries: 3
|
|
||||||
delay: 2
|
|
||||||
until: scp_put.rc == 0
|
|
||||||
|
|
||||||
############ step 2
|
|
||||||
|
|
||||||
- name: Compute MD5 of local wifidebug.sh
|
|
||||||
delegate_to: localhost
|
|
||||||
command: md5sum files/wifidebug.sh
|
|
||||||
register: md5_local_wdbg
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Compute MD5 of remote /root/wifidebug.sh
|
|
||||||
raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
|
|
||||||
register: md5_remote_wdbg
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
- name: Decide if wifidebug.sh needs upload
|
|
||||||
set_fact:
|
|
||||||
upload_wdbg: >-
|
|
||||||
{{ (md5_remote_wdbg.rc != 0)
|
|
||||||
or ((md5_local_wdbg.stdout.split()[0])
|
|
||||||
!= (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
|
|
||||||
|
|
||||||
- name: Upload wifidebug.sh via scp (overwrite if changed)
|
|
||||||
when: upload_wdbg | bool
|
|
||||||
delegate_to: localhost
|
|
||||||
command: >
|
|
||||||
sshpass -p {{ ssh_pass | quote }}
|
|
||||||
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
||||||
files/wifidebug.sh
|
|
||||||
{{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
|
|
||||||
register: scp_wifidebug
|
|
||||||
retries: 3
|
|
||||||
delay: 2
|
|
||||||
until: scp_wifidebug.rc == 0
|
|
||||||
|
|
||||||
- name: Ensure /root/wifidebug.sh is executable and owned by root
|
|
||||||
raw: |
|
|
||||||
chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
|
|
||||||
|
|
||||||
############ step 3
|
|
||||||
|
|
||||||
- name: Ensure /etc/crontabs/root exists (touch with perms)
|
|
||||||
raw: |
|
|
||||||
if [ ! -f /etc/crontabs/root ]; then
|
|
||||||
touch /etc/crontabs/root
|
|
||||||
fi
|
|
||||||
chown root:root /etc/crontabs/root
|
|
||||||
chmod 0644 /etc/crontabs/root
|
|
||||||
|
|
||||||
- name: Check if cron line already present
|
|
||||||
raw: |
|
|
||||||
grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
|
|
||||||
register: cron_grep
|
|
||||||
failed_when: false
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Upload snippet crond-root to /tmp (only if missing)
|
|
||||||
when: cron_grep.rc != 0
|
|
||||||
delegate_to: localhost
|
|
||||||
command: >
|
|
||||||
sshpass -p {{ ssh_pass | quote }}
|
|
||||||
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
||||||
files/crond-root
|
|
||||||
{{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
|
|
||||||
|
|
||||||
- name: Append snippet to /etc/crontabs/root (only if missing)
|
|
||||||
when: cron_grep.rc != 0
|
|
||||||
raw: |
|
|
||||||
cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
|
|
||||||
register: cron_append
|
|
||||||
changed_when: true
|
|
||||||
|
|
||||||
############ step 4
|
|
||||||
|
|
||||||
- name: Skip config.json edit if remote_syslog already matches
|
|
||||||
raw: |
|
|
||||||
EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
|
|
||||||
SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
|
|
||||||
if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
|
|
||||||
echo "SKIP"
|
|
||||||
else
|
|
||||||
echo "EDIT"
|
|
||||||
fi
|
|
||||||
register: remote_syslog_check
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Set do_edit flag from skip check
|
|
||||||
set_fact:
|
|
||||||
do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
|
|
||||||
|
|
||||||
- name: Fetch /tmp/config.json to controller
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
command: >
|
|
||||||
sshpass -p {{ ssh_pass | quote }}
|
|
||||||
scp -q -o StrictHostKeyChecking=no
|
|
||||||
{{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
|
|
||||||
/tmp/{{ inventory_hostname }}_config.json
|
|
||||||
|
|
||||||
- name: Strict in-place style-preserving edit (services.remote_syslog only)
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
shell: |
|
|
||||||
awk '
|
|
||||||
BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
|
|
||||||
{
|
|
||||||
line = $0
|
|
||||||
if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
|
|
||||||
if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
|
|
||||||
if (in_rs) {
|
|
||||||
if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
|
|
||||||
line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
|
|
||||||
done_enabled=1
|
|
||||||
}
|
|
||||||
if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
|
|
||||||
line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
|
|
||||||
done_server=1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
print line
|
|
||||||
if (in_rs) {
|
|
||||||
opens = gsub(/{/, "{", $0)
|
|
||||||
closes = gsub(/}/, "}", $0)
|
|
||||||
depth += (opens - closes)
|
|
||||||
if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
' "/tmp/{{ inventory_hostname }}_config.json" \
|
|
||||||
> "/tmp/{{ inventory_hostname }}_config.json.new"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
|
|
||||||
- name: Push edited config.json.new to remote temp
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
shell: |
|
|
||||||
sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
|
|
||||||
"/tmp/{{ inventory_hostname }}_config.json.new" \
|
|
||||||
{{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
|
|
||||||
- name: Compute MD5 of local .new
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
|
|
||||||
register: md5_local
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Compute MD5 of remote .new (no Python)
|
|
||||||
when: do_edit
|
|
||||||
raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
|
|
||||||
register: md5_remote
|
|
||||||
changed_when: false
|
|
||||||
|
|
||||||
- name: Fail if MD5 mismatch
|
|
||||||
when:
|
|
||||||
- do_edit
|
|
||||||
- (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
|
|
||||||
fail:
|
|
||||||
msg: "MD5 mismatch between controller and remote copy — aborting replace."
|
|
||||||
|
|
||||||
- name: Commit new config.json (overwrite original)
|
|
||||||
when: do_edit
|
|
||||||
raw: |
|
|
||||||
cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
|
|
||||||
mv /tmp/config.json.new /tmp/config.json
|
|
||||||
|
|
||||||
- name: Apply config to runtime (sysconf -w)
|
|
||||||
when: do_edit
|
|
||||||
raw: |
|
|
||||||
sysconf -w || /usr/sbin/sysconf -w
|
|
||||||
register: sysconf_result
|
|
||||||
changed_when: true
|
|
||||||
failed_when: sysconf_result.rc not in [0]
|
|
||||||
|
|
||||||
# (d) Pre-restart notice to control queue
|
|
||||||
- name: Build pre-restart journal (explain why)
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
set_fact:
|
|
||||||
prereboot_payload:
|
|
||||||
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
|
|
||||||
|
|
||||||
- name: Publish pre-restart journal to control queue
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ prereboot_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_preboot
|
|
||||||
changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
|
|
||||||
|
|
||||||
- name: Trigger full restart (system-stop; system-start) from controller with 10s cap
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: restart_kick
|
|
||||||
changed_when: true
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
|
|
||||||
# probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
|
|
||||||
- name: Probe SSH with nc (24 tries, 5s each)
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
shell: "nc -z -w 3 {{ ansible_host }} 22"
|
|
||||||
register: nc_probe
|
|
||||||
retries: 24
|
|
||||||
delay: 5
|
|
||||||
until: nc_probe.rc == 0
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false
|
|
||||||
|
|
||||||
- name: Set ssh_up fact (from nc probe)
|
|
||||||
when: do_edit
|
|
||||||
delegate_to: localhost
|
|
||||||
set_fact:
|
|
||||||
ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
|
|
||||||
|
|
||||||
- name: Fail if SSH did not return after restart
|
|
||||||
when:
|
|
||||||
- do_edit
|
|
||||||
- not ssh_up | bool
|
|
||||||
fail:
|
|
||||||
msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
|
|
||||||
|
|
||||||
# ======== END OF SURGICAL CHANGE ========
|
|
||||||
|
|
||||||
- name: Set result status (success deployed or no change)
|
|
||||||
set_fact:
|
|
||||||
result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
|
|
||||||
|
|
||||||
when: ping_result is succeeded
|
|
||||||
|
|
||||||
- name: Set status fact (no ssh)
|
|
||||||
when: ping_result is failed
|
|
||||||
set_fact:
|
|
||||||
result_status: "NO_SSH"
|
|
||||||
|
|
||||||
rescue:
|
|
||||||
- name: Mark result as failed
|
|
||||||
set_fact:
|
|
||||||
result_status: "FAILED during {{ ansible_failed_task.name }}"
|
|
||||||
|
|
||||||
always:
|
|
||||||
|
|
||||||
- name: Compute inscope device
|
|
||||||
set_fact:
|
|
||||||
inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
|
|
||||||
|
|
||||||
# (a) Set custom field wifidebug -> v15 on control queue
|
|
||||||
|
|
||||||
- name: Build custom-field payload (wifidebug -> version)
|
|
||||||
delegate_to: localhost
|
|
||||||
set_fact:
|
|
||||||
wdbg_cf_payload:
|
|
||||||
inscope_device: "{{ inscope_device_name }}"
|
|
||||||
task_name: "deploy_wifidebug"
|
|
||||||
task_result: "{{ wdbg_version }}"
|
|
||||||
|
|
||||||
- name: Publish custom-field update to control queue
|
|
||||||
delegate_to: localhost
|
|
||||||
uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ wdbg_cf_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_cf
|
|
||||||
changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
|
|
||||||
|
|
||||||
- name: Build actions list
|
|
||||||
set_fact:
|
|
||||||
_actions_list: >-
|
|
||||||
{{
|
|
||||||
[]
|
|
||||||
+ ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], []))
|
|
||||||
+ (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
|
|
||||||
+ ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
|
|
||||||
+ ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
|
|
||||||
+ ((do_edit | default(false) | bool) | ternary(['restarted services'], []))
|
|
||||||
}}
|
|
||||||
|
|
||||||
- name: Build actions string
|
|
||||||
set_fact:
|
|
||||||
_actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
|
|
||||||
|
|
||||||
|
|
||||||
- name: Build wrap-up journal payload
|
|
||||||
delegate_to: localhost
|
|
||||||
set_fact:
|
|
||||||
wrap_payload:
|
|
||||||
inscope_device: "{{ inscope_device_name }}"
|
|
||||||
task_name: "journal_add"
|
|
||||||
task_result: >-
|
|
||||||
wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
|
|
||||||
— actions: {{ _actions_str }}
|
|
||||||
|
|
||||||
- name: Publish wrap-up journal to control queue
|
|
||||||
delegate_to: localhost
|
|
||||||
uri:
|
|
||||||
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
||||||
method: POST
|
|
||||||
user: "{{ rmq_user }}"
|
|
||||||
password: "{{ rmq_pass }}"
|
|
||||||
force_basic_auth: true
|
|
||||||
status_code: 200
|
|
||||||
headers:
|
|
||||||
content-type: "application/json"
|
|
||||||
body_format: json
|
|
||||||
body:
|
|
||||||
properties:
|
|
||||||
content_type: "application/json"
|
|
||||||
routing_key: "{{ control_queue }}"
|
|
||||||
payload: "{{ wrap_payload | to_json }}"
|
|
||||||
payload_encoding: "string"
|
|
||||||
register: rmq_wrap
|
|
||||||
changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
|
|
||||||
|
|
||||||
# Local summary (kept for operator visibility)
|
|
||||||
- name: Summary
|
|
||||||
debug:
|
|
||||||
msg:
|
|
||||||
- "result_status: {{ result_status }}"
|
|
||||||
- "we're good"
|
|
||||||
|
|
||||||
Reference in New Issue
Block a user