pavel/ansible-worker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0509

Browse Source
This commit is contained in:
pavel
2025-10-23 05:09:48 +03:00
parent f0d0fe9760
commit 9aadf7bb10
14 changed files with 4 additions and 8982 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
files/ansible-playbooks/afterupgrade_check.yml-from_chat
View File

@@ -1,355 +0,0 @@
---
- name: After-upgrade verification (banner check + reporting)
hosts: all
gather_facts: no
# RabbitMQ + defaults (match the big script)
vars:
rmq_host: "10.210.12.2"
rmq_port: 15672
rmq_user: "admin"
rmq_pass: "change_me"
rmq_vhost: "app"
rmq_exchange: "controls"
control_queue: "queue_controls"
# Probing/SSH defaults
tcp_port: 22
nc_timeout: 5
ssh_user: "{{ ansible_user | default('root') }}"
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
ssh_timeout: 10
# Do NOT self-reference max_attempts. We’ll normalize below.
max_attempts_default: 3
tasks:
# ---- Normalize metadata safely (no self-referential defaults) ----
- name: Normalize metadata (no clever transforms)
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
target_version: "{{ target_version | default('') }}"
# preserve the original string verbatim for all subsequent retries
target_version_full: "{{ target_version | default('') }}"
- name: Show received metadata
ansible.builtin.debug:
msg:
- "attempt={{ attempt }}"
- "max_attempts={{ effective_max_attempts }}"
- "correlation_id={{ correlation_id }}"
- "original_emitted_at={{ original_emitted_at }}"
- "target_version(full)={{ target_version_full }}"
# ---- Fast TCP reachability probe (controller-side) ----
- name: Check if TCP/{{ tcp_port }} is reachable with nc
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
register: nc_probe
changed_when: false
ignore_errors: true
- name: Build failure journal (no TCP connectivity) + mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (no TCP connectivity)
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
- name: Stop host after TCP failure (we’ll schedule or close out below)
when: nc_probe.rc != 0
ansible.builtin.meta: noop
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
- name: Probe banner via SSH from controller (classic extraction)
when: nc_probe.rc == 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
USER="{{ ssh_user }}"
HOST="{{ ansible_host | default(inventory_hostname) }}"
sshpass -p '{{ ssh_pass }}' \
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
"${USER}@${HOST}" \
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
register: banner_probe
changed_when: false
ignore_errors: true
- name: Show the current version (banner line)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.debug:
msg: "{{ banner_probe.stdout | trim }}"
- name: Evaluate version match (full-string contains check)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.set_fact:
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
# ---- Journaling paths ----
# Success: banner matches expected full target_version
- name: Build success journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_success_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish success journal to control queue
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_success_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_success
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
# NEW: send a control tag to clean up device state on success
- name: Build cleanup control payload (update_cleanup_success)
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
control_cleanup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "update_cleanup_success"
- name: Publish cleanup control message to control queue
when: control_cleanup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ control_cleanup_payload | to_json }}"
payload_encoding: "string"
register: rmq_cleanup_success
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
# Mismatch: reachable & banner read, but not equal to target_version
- name: Build mismatch journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_mismatch_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish mismatch journal to control queue
when: journal_mismatch_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_mismatch_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_mismatch
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
# SSH error path: TCP OK, but SSH failed
- name: Build failure journal payload (ssh error) + mark retry
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
journal_fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
{{ (banner_probe.stderr | default('') | trim) }}
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (ssh error) to control queue
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_fail
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
- name: Compute next-attempt delay (ms) according to policy
when: (_needs_retry | default(false)) | bool
ansible.builtin.set_fact:
next_attempt: "{{ attempt | int + 1 }}"
next_delay_sec: >-
{% if attempt | int == 1 %}
300
{% elif attempt | int == 2 %}
600
{% else %}
0
{% endif %}
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
# If we've reached the cap, send a final “gave up” journal and stop.
- name: Build final gave-up journal (max attempts reached)
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
journal_gaveup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
- name: Publish final gave-up journal
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_gaveup_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_gaveup
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
- name: Stop host after final gave-up
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
# Otherwise schedule the next attempt (only if we still have budget)
- name: Build delayed after-upgrade payload for next attempt
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
delayed_payload:
task_name: "afterupgrade_check"
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
attempt: "{{ next_attempt | int }}"
max_attempts: "{{ effective_max_attempts | int }}"
correlation_id: "{{ correlation_id }}"
original_emitted_at: "{{ original_emitted_at }}"
target_version: "{{ target_version_full }}"
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
expiration: "{{ (next_delay_ms | int) | string }}"
correlation_id: "{{ correlation_id }}"
routing_key: "deviceconfig"
payload: "{{ delayed_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_next
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
when: (_needs_retry | default(false)) | bool
ansible.builtin.meta: end_host

432
files/ansible-playbooks/afterupgrade_check.yml-from_container
View File

@@ -1,432 +0,0 @@
---
- name: After-upgrade verification (banner check + reporting)
hosts: all
gather_facts: no
# RabbitMQ + defaults (match the big script)
vars:
rmq_host: "10.210.12.2"
rmq_port: 15672
rmq_user: "admin"
rmq_pass: "change_me"
rmq_vhost: "app"
rmq_exchange: "controls"
control_queue: "queue_controls"
# Probing/SSH defaults
tcp_port: 22
nc_timeout: 5
ssh_user: "{{ ansible_user | default('root') }}"
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
ssh_timeout: 10
# Do NOT self-reference max_attempts. We’ll normalize below.
max_attempts_default: 3
tasks:
# ---- Normalize metadata safely (no self-referential defaults) ----
- name: Normalize metadata (no clever transforms)
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
target_version: "{{ target_version | default('') }}"
# preserve the original string verbatim for all subsequent retries
target_version_full: "{{ target_version | default('') }}"
- name: Show received metadata
ansible.builtin.debug:
msg:
- "attempt={{ attempt }}"
- "max_attempts={{ effective_max_attempts }}"
- "correlation_id={{ correlation_id }}"
- "original_emitted_at={{ original_emitted_at }}"
- "target_version(full)={{ target_version_full }}"
# ---- Fast TCP reachability probe (controller-side) ----
- name: Check if TCP/{{ tcp_port }} is reachable with nc
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
register: nc_probe
changed_when: false
ignore_errors: true
- name: Build failure journal (no TCP connectivity) + mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (no TCP connectivity)
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
- name: Stop host after TCP failure (we’ll schedule or close out below)
when: nc_probe.rc != 0
ansible.builtin.meta: noop
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
- name: Probe banner via SSH from controller (classic extraction)
when: nc_probe.rc == 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
USER="{{ ssh_user }}"
HOST="{{ ansible_host | default(inventory_hostname) }}"
sshpass -p '{{ ssh_pass }}' \
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
"${USER}@${HOST}" \
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
register: banner_probe
changed_when: false
ignore_errors: true
- name: Show the current version (banner line)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.debug:
msg: "{{ banner_probe.stdout | trim }}"
- name: Evaluate version match (full-string contains check)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.set_fact:
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
# ---- Journaling paths ----
# Success: banner matches expected full target_version
- name: Build success journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_success_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
Banner='{{ (banner_probe.stdout | default('') | trim) }}'
Target='{{ target_version_full }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish success journal to control queue
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_success_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_success
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
# NEW: send a control tag to clean up device state on success
- name: Build cleanup control payload (update_cleanup_success)
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
control_cleanup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "update_cleanup_success"
- name: Publish cleanup control message to control queue
when: control_cleanup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ control_cleanup_payload | to_json }}"
payload_encoding: "string"
register: rmq_cleanup_success
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
# --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag ---
- name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801")
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.set_fact:
indoor_fwver_norm: >-
{{
( (dev2_fwver.stdout | default('') | trim)
| regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1')
| regex_replace('\\s+', ' ')
| trim
)
}}
- name: Publish NetBox custom field indoor_fwver to control queue
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers: { content-type: "application/json" }
body_format: json
body:
properties: { content_type: "application/json" }
routing_key: "{{ control_queue }}"
payload: "{{ {
'inscope_device': (ansible_hostname | default(inventory_hostname)),
'task_name': 'custom_field_set',
'task_add1': 'indoor_fwver',
'task_result': (indoor_fwver_norm | default(''))
} | to_json }}"
payload_encoding: "string"
register: rmq_cf_indoor_fwver
changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool)
- name: Build payload to remove indoor-restart-scheduled tag (DEV1)
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.set_fact:
tag_remove_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "tag_remove"
task_result: "indoor-restart-scheduled"
- name: Publish indoor-restart-scheduled tag removal
when: tag_remove_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ tag_remove_payload | to_json }}"
payload_encoding: "string"
register: rmq_tag_remove_sched
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
# Mismatch: reachable & banner read, but not equal to target_version
- name: Build mismatch journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_mismatch_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish mismatch journal to control queue
when: journal_mismatch_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_mismatch_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_mismatch
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
# SSH error path: TCP OK, but SSH failed
- name: Build failure journal payload (ssh error) + mark retry
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
journal_fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
{{ (banner_probe.stderr | default('') | trim) }}
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (ssh error) to control queue
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_fail
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
- name: Compute next-attempt delay (ms) according to policy
when: (_needs_retry | default(false)) | bool
ansible.builtin.set_fact:
next_attempt: "{{ attempt | int + 1 }}"
next_delay_sec: >-
{% if attempt | int == 1 %}
300
{% elif attempt | int == 2 %}
600
{% else %}
0
{% endif %}
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
# If we've reached the cap, send a final “gave up” journal and stop.
- name: Build final gave-up journal (max attempts reached)
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
journal_gaveup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
- name: Publish final gave-up journal
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_gaveup_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_gaveup
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
- name: Stop host after final gave-up
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
# Otherwise schedule the next attempt (only if we still have budget)
- name: Build delayed after-upgrade payload for next attempt
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
delayed_payload:
task_name: "afterupgrade_check"
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
attempt: "{{ next_attempt | int }}"
max_attempts: "{{ effective_max_attempts | int }}"
correlation_id: "{{ correlation_id }}"
original_emitted_at: "{{ original_emitted_at }}"
target_version: "{{ target_version_full }}"
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
expiration: "{{ (next_delay_ms | int) | string }}"
correlation_id: "{{ correlation_id }}"
routing_key: "deviceconfig"
payload: "{{ delayed_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_next
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
when: (_needs_retry | default(false)) | bool
ansible.builtin.meta: end_host

359
files/ansible-playbooks/afterupgrade_check.yml-screewed
View File

@@ -1,359 +0,0 @@
---
- name: After-upgrade verification (banner check + reporting)
hosts: all
gather_facts: no
# RabbitMQ + defaults (match the big script)
vars:
rmq_host: "10.210.12.2"
rmq_port: 15672
rmq_user: "admin"
rmq_pass: "change_me"
rmq_vhost: "app"
rmq_exchange: "controls"
control_queue: "queue_controls"
# Probing/SSH defaults
tcp_port: 22
nc_timeout: 5
ssh_user: "{{ ansible_user | default('root') }}"
ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
ssh_timeout: 10
# Do NOT self-reference max_attempts. We’ll normalize below.
max_attempts_default: 3
tasks:
# ---- Normalize metadata safely (no self-referential defaults) ----
- name: Normalize metadata (no clever transforms)
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
target_version: "{{ target_version | default('') }}"
# preserve the original string verbatim for all subsequent retries
target_version_full: "{{ target_version | default('') }}"
- name: Show received metadata
ansible.builtin.debug:
msg:
- "attempt={{ attempt }}"
- "max_attempts={{ effective_max_attempts }}"
- "correlation_id={{ correlation_id }}"
- "original_emitted_at={{ original_emitted_at }}"
- "target_version(full)={{ target_version_full }}"
# ---- Fast TCP reachability probe (controller-side) ----
- name: Check if TCP/{{ tcp_port }} is reachable with nc
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
register: nc_probe
changed_when: false
ignore_errors: true
- name: Build failure journal (no TCP connectivity) + mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }}
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (no TCP connectivity)
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
# If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
- name: Stop host after TCP failure (we’ll schedule or close out below)
when: nc_probe.rc != 0
ansible.builtin.meta: noop
# ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
- name: Probe banner via SSH from controller (classic extraction)
when: nc_probe.rc == 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
USER="{{ ssh_user }}"
HOST="{{ ansible_host | default(inventory_hostname) }}"
sshpass -p '{{ ssh_pass }}' \
ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
"${USER}@${HOST}" \
"PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
register: banner_probe
changed_when: false
ignore_errors: true
- name: Show the current version (banner line)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.debug:
msg: "{{ banner_probe.stdout | trim }}"
- name: Evaluate version match (full-string contains check)
when: nc_probe.rc == 0 and banner_probe.rc == 0
ansible.builtin.set_fact:
version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}"
# ---- Journaling paths ----
# Success: banner matches expected full target_version
- name: Build success journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_success_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
Banner='{{ (banner_probe.stdout | default('') | trim) }}'
Target='{{ target_version_full }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish success journal to control queue
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_success_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_success
changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)
# NEW: send a control tag to clean up device state on success
- name: Build cleanup control payload (update_cleanup_success)
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.set_fact:
control_cleanup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "update_cleanup_success"
- name: Publish cleanup control message to control queue
when: control_cleanup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ control_cleanup_payload | to_json }}"
payload_encoding: "string"
register: rmq_cleanup_success
changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)
# Mismatch: reachable & banner read, but not equal to target_version
- name: Build mismatch journal payload
when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_mismatch_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
Correlation={{ correlation_id }} Original={{ original_emitted_at }}
- name: Publish mismatch journal to control queue
when: journal_mismatch_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_mismatch_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_mismatch
changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)
# SSH error path: TCP OK, but SSH failed
- name: Build failure journal payload (ssh error) + mark retry
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
journal_fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
{{ (banner_probe.stderr | default('') | trim) }}
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (ssh error) to control queue
when: nc_probe.rc == 0 and banner_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_fail
changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)
# ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
- name: Compute next-attempt delay (ms) according to policy
when: (_needs_retry | default(false)) | bool
ansible.builtin.set_fact:
next_attempt: "{{ attempt | int + 1 }}"
next_delay_sec: >-
{% if attempt | int == 1 %}
300
{% elif attempt | int == 2 %}
600
{% else %}
0
{% endif %}
next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"
# If we've reached the cap, send a final “gave up” journal and stop.
- name: Build final gave-up journal (max attempts reached)
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
journal_gaveup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
- name: Publish final gave-up journal
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_gaveup_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_gaveup
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
- name: Stop host after final gave-up
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
# Otherwise schedule the next attempt (only if we still have budget)
- name: Build delayed after-upgrade payload for next attempt
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
delayed_payload:
task_name: "afterupgrade_check"
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
attempt: "{{ next_attempt | int }}"
max_attempts: "{{ effective_max_attempts | int }}"
correlation_id: "{{ correlation_id }}"
original_emitted_at: "{{ original_emitted_at }}"
target_version: "{{ target_version_full }}"
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
expiration: "{{ (next_delay_ms | int) | string }}"
correlation_id: "{{ correlation_id }}"
routing_key: "deviceconfig"
payload: "{{ delayed_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_next
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
- name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
when: (_needs_retry | default(false)) | bool
ansible.builtin.meta: end_host

7
files/ansible-playbooks/afterupgrade_indoor_check.yml
View File

@@ -30,6 +30,8 @@
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
debug_aic: false
tasks:
# ---- Normalize metadata from delayed message ----
- name: Normalize after upgrade metadata into facts
@@ -353,13 +355,12 @@
- name: Debug | banner normalization inputs
delegate_to: localhost
when: debug_aic | bool
when: (debug_aic | default(false)) | bool
ansible.builtin.debug:
msg:
- "banner_raw={{ (firmware_banner.stdout | default(firmware_banner) | default('')) | trim }}"
- "banner_raw={{ (dev2_fwver.stdout | default('') | trim) }}"
- "normalize rule: '2.2.1 rev 6801' -> '2.2.1-r6801'"
- name: Debug show raw firmware readout from DEV2
delegate_to: localhost
ansible.builtin.debug:

201
files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries
View File

@@ -1,201 +0,0 @@
---
- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
hosts: all
gather_facts: no
vars:
# ---------------- BusyBox-safe path prefix ----------------
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
# ---------------- DEV1 (outer device) ----------------
dev1_user: "root"
dev1_pass: "wavewave"
dev1_iface: "br-wan"
# ---------------- DEV2 (indoor behind DEV1) ----------------
dev2_host: "192.168.1.1"
dev2_port: 22
dev2_side_ip: "192.168.1.11/24"
dev2_ssh_user: "root"
dev2_passfiles:
- "basicpass"
- "basicpass2"
# ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
tasks:
# ---------------- Step 1: Read DEV1 hostname and sanity ----------------
- name: Read DEV1 hostname
ansible.builtin.raw: >
{{ pathprefix }}
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
register: dev1_host_read
changed_when: false
- name: Stop if DEV1 hostname mismatch
ansible.builtin.meta: end_host
when: (dev1_host_read.stdout | trim | length > 0) and
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
# ---------------- Step 2: Setup temporary IP for reachability ----------------
- name: Add temporary IP on DEV1 (ignore if exists)
ansible.builtin.raw: >
{{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
register: add_ip
changed_when: add_ip.rc == 0
failed_when: >
add_ip.rc != 0 and
('File exists' not in (add_ip.stdout | default(''))) and
('File exists' not in (add_ip.stderr | default('')))
# ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
- name: Pick random free local port
delegate_to: localhost
ansible.builtin.shell: |
set -e
for i in $(seq 1 25); do
p="$(shuf -i 20000-39999 -n1)"
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
done
exit 1
register: pick_port
changed_when: false
- name: Stop if no free local port
ansible.builtin.meta: end_host
when: (pick_port.stdout | trim | length) == 0
- name: Record chosen local port and control dir
delegate_to: localhost
ansible.builtin.set_fact:
_local_port: "{{ pick_port.stdout | trim }}"
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
- name: Build tunnel control socket path
delegate_to: localhost
ansible.builtin.set_fact:
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
- name: Start SSH ControlMaster tunnel via DEV1
delegate_to: localhost
ansible.builtin.shell: |
set -e
sshpass -p '{{ dev1_pass }}' ssh -f -N \
-M -S "{{ _ctrl_sock }}" \
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
"{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
args:
executable: /bin/bash
register: start_tunnel
changed_when: true
# ---------------- Step 4: Determine working password for DEV2 ----------------
- name: Try both passfiles for DEV2
delegate_to: localhost
ansible.builtin.shell: |
for f in {{ dev2_passfiles | join(' ') }}; do
if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
echo "$f"; exit 0;
fi
done
echo NONE
register: dev2_passfile_try
changed_when: false
- name: Save selected DEV2 passfile
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
changed_when: false
- name: Stop if no valid passfile found
ansible.builtin.meta: end_host
when: dev2_passfile_used == "NONE"
# ---------------- Step 5: Read firmware version on DEV2 ----------------
- name: Read firmware version from DEV2
delegate_to: localhost
ansible.builtin.shell: |
sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
register: dev2_fwver
changed_when: false
ignore_errors: true
- name: Show firmware version readout
delegate_to: localhost
ansible.builtin.debug:
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
# ---------------- Step 6: Publish result journal ----------------
- name: Build journal payload
delegate_to: localhost
ansible.builtin.set_fact:
journal_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
tunnel_port={{ _local_port }}
- name: Publish journal to control queue
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_fwver
changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool)
post_tasks:
- name: Cleanup tunnel and temp IP
block:
- ansible.builtin.debug:
msg: "Cleaning up tunnel + temp IP"
changed_when: false
delegate_to: localhost
always:
- name: Close tunnel
delegate_to: localhost
ansible.builtin.shell: |
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
changed_when: false
ignore_errors: true
- name: Remove control dir
delegate_to: localhost
ansible.builtin.file:
path: "{{ _ctrl_dir | default('/tmp/none') }}"
state: absent
ignore_errors: true
- name: Remove temporary IP from DEV1
ansible.builtin.raw: >
{{ pathprefix }}
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
register: del_ip
failed_when: false
changed_when: false

0
files/ansible-playbooks/afterupgrade_indoor_check.yml-chat
View File

644
files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors
View File

@@ -1,644 +0,0 @@
---
- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel)
hosts: all
gather_facts: no
vars:
# ---------------- BusyBox-safe path prefix ----------------
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
# ---------------- DEV1 (outer device) ----------------
dev1_user: "root"
dev1_pass: "wavewave"
dev1_iface: "br-wan"
# ---------------- DEV2 (indoor behind DEV1) ----------------
dev2_host: "192.168.1.1"
dev2_port: 22
dev2_side_ip: "192.168.1.11/24"
dev2_ssh_user: "root"
dev2_passfiles:
- "basicpass"
- "basicpass2"
# ---------------- RabbitMQ (same env scheme as main playbooks) ----------------
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
tasks:
# ---- Normalize metadata (from delayed message) ----
- name: Normalize after-upgrade metadata
delegate_to: localhost
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
target_version: "{{ target_version | default('') }}"
target_version_full: "{{ target_version | default('') }}"
# ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ----
- name: Check if TCP/22 on DEV1 is reachable
delegate_to: localhost
ansible.builtin.shell: |
nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22
register: nc_probe
changed_when: false
ignore_errors: true
# ---- If TCP down: journal + schedule next try or give up ----
- name: Build failure journal (TCP unreachable) + mark retry
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.set_fact:
fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}):
TCP 22 unreachable. Correlation={{ correlation_id }}
Original={{ original_emitted_at }} Target='{{ target_version_full }}'
_needs_retry: true
- name: Publish failure journal (TCP unreachable)
when: nc_probe.rc != 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers: { content-type: "application/json" }
body_format: json
body:
properties: { content_type: "application/json" }
routing_key: "{{ control_queue }}"
payload: "{{ fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_tcp_fail
changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)
- name: Compute next-attempt delay (10 minutes) and counters
when: (_needs_retry | default(false)) | bool
delegate_to: localhost
ansible.builtin.set_fact:
next_attempt: "{{ attempt | int + 1 }}"
next_delay_sec: 600
next_delay_ms: 600000
- name: Build final gave-up journal (max attempts reached)
when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
journal_gaveup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
Exhausted attempts. Last error path=TCP.
Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
- name: Publish final gave-up journal
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers: { content-type: "application/json" }
body_format: json
body:
properties: { content_type: "application/json" }
routing_key: "{{ control_queue }}"
payload: "{{ journal_gaveup_payload | to_json }}"
payload_encoding: "string"
register: rmq_j_gaveup
changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)
- name: Stop host after final gave-up
when: journal_gaveup_payload is defined
ansible.builtin.meta: end_host
- name: Build delayed after-upgrade payload for next attempt (10m)
when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
delegate_to: localhost
ansible.builtin.set_fact:
delayed_payload:
task_name: "afterupgrade_indoor_check"
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
attempt: "{{ next_attempt | int }}"
max_attempts: "{{ effective_max_attempts | int }}"
correlation_id: "{{ correlation_id }}"
original_emitted_at: "{{ original_emitted_at }}"
target_version: "{{ target_version_full }}"
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next attempt to holding (dead-letters to deviceconfig)
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers: { content-type: "application/json" }
body_format: json
body:
properties:
content_type: "application/json"
expiration: "{{ (next_delay_ms | int) | string }}"
correlation_id: "{{ correlation_id }}"
routing_key: "deviceconfig"
payload: "{{ delayed_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_next
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
- name: Stop host after scheduling next attempt
when: (_needs_retry | default(false)) | bool
ansible.builtin.meta: end_host
# ---------------- Step 1: Read DEV1 hostname and sanity ----------------
- name: Read DEV1 hostname
when: nc_probe.rc == 0
ansible.builtin.raw: >
{{ pathprefix }}
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
register: dev1_host_read
changed_when: false
- name: Stop if DEV1 hostname mismatch
ansible.builtin.meta: end_host
when: (dev1_host_read.stdout | trim | length > 0) and
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
# ---------------- Step 2: Setup temporary IP for reachability ----------------
- name: Add temporary IP on DEV1 (ignore if exists)
ansible.builtin.raw: >
{{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
register: add_ip
changed_when: add_ip.rc == 0
failed_when: >
add_ip.rc != 0 and
('File exists' not in (add_ip.stdout | default(''))) and
('File exists' not in (add_ip.stderr | default('')))
# ---------------- Step 3: Start SSH tunnel via DEV1 ----------------
- name: Pick random free local port
delegate_to: localhost
ansible.builtin.shell: |
set -e
for i in $(seq 1 25); do
p="$(shuf -i 20000-39999 -n1)"
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi
done
exit 1
register: pick_port
changed_when: false
- name: Stop if no free local port
ansible.builtin.meta: end_host
when: (pick_port.stdout | trim | length) == 0
- name: Record chosen local port and control dir
delegate_to: localhost
ansible.builtin.set_fact:
_local_port: "{{ pick_port.stdout | trim }}"
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
- name: Build tunnel control socket path
delegate_to: localhost
ansible.builtin.set_fact:
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
- name: Start SSH ControlMaster tunnel via DEV1
delegate_to: localhost
ansible.builtin.shell: |
set -e
sshpass -p '{{ dev1_pass }}' ssh -f -N \
-M -S "{{ _ctrl_sock }}" \
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \
"{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}"
args:
executable: /bin/bash
register: start_tunnel
changed_when: true
# --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) ---
- name: Small delay for tunnel to settle
delegate_to: localhost
ansible.builtin.wait_for:
timeout: 1
changed_when: false
- name: Verify tunnel master running (ssh -O check)
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ ansible_host | default(inventory_hostname) }}"
ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
register: tun_check
changed_when: false
- name: Debug tunnel check
delegate_to: localhost
ansible.builtin.debug:
msg:
- "tunnel_check.rc={{ tun_check.rc }}"
- "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
- name: Sanity confirm tunnel TCP reachability to DEV2
delegate_to: localhost
ansible.builtin.shell: |
set -e
nc -z -w5 127.0.0.1 "{{ _local_port }}"
register: nc_probe
changed_when: false
ignore_errors: true
- name: Debug reachability result
delegate_to: localhost
ansible.builtin.debug:
msg:
- "nc.rc={{ nc_probe.rc }}"
- "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
- "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
- name: Stop if tunnel TCP check failed
ansible.builtin.meta: end_host
when: nc_probe.rc != 0
# ---------------- Step 4: Determine working password for DEV2 ----------------
- name: Try both passfiles for DEV2
delegate_to: localhost
ansible.builtin.shell: |
for f in {{ dev2_passfiles | join(' ') }}; do
if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then
echo "$f"; exit 0;
fi
done
echo NONE
register: dev2_passfile_try
changed_when: false
- name: Save selected DEV2 passfile
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}"
changed_when: false
- name: Debug selected DEV2 passfile
delegate_to: localhost
ansible.builtin.debug:
msg: "dev2_passfile_used={{ dev2_passfile_used }}"
- name: Stop if no valid passfile found
ansible.builtin.meta: end_host
when: dev2_passfile_used == "NONE"
# ---------------- Step 5: Read firmware version on DEV2 ----------------
- name: Read firmware version from DEV2
delegate_to: localhost
ansible.builtin.shell: |
sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \
root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown"
register: dev2_fwver
changed_when: false
ignore_errors: true
- name: Show firmware version readout
delegate_to: localhost
ansible.builtin.debug:
msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}"
# ---------------- Retry metadata + success evaluation ----------------
- name: Normalize retry metadata for indoor checker
ansible.builtin.set_fact:
attempt: "{{ (attempt | default(1)) | int }}"
effective_max_attempts: "{{ (max_attempts | default(3)) | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
- name: Evaluate indoor success (firmware read OK)
delegate_to: localhost
ansible.builtin.set_fact:
indoor_success: "{{ (dev2_fwver.rc | default(1) == 0)
and ((dev2_fwver.stdout | default('') | trim) | length > 0)
and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}"
# ---------------- Step 6: Publish result journal (+ tags on success) ----------------
# SUCCESS PATH: add tag, remove scheduled tag, and journal with "success"
- name: Build success journal payload
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.set_fact:
journal_success_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}'
tunnel_port={{ _local_port }}
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish success journal to control queue
when: journal_success_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_success_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_success
changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool)
- name: Build payload to add indoor-update-success tag (DEV1)
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.set_fact:
tag_add_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "tag_add"
task_result: "indoor-update-success"
- name: Publish indoor-update-success tag
when: tag_add_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ tag_add_payload | to_json }}"
payload_encoding: "string"
register: rmq_tag_add_success
changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool)
- name: Build payload to remove indoor-restart-scheduled tag (DEV1)
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.set_fact:
tag_remove_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "tag_remove"
task_result: "indoor-restart-scheduled"
- name: Publish indoor-restart-scheduled tag removal
when: tag_remove_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ tag_remove_payload | to_json }}"
payload_encoding: "string"
register: rmq_tag_remove_sched
changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool)
# --- Normalize firmware string and set custom field on success ---
- name: Capture raw firmware banner from DEV2 (for normalization)
when: indoor_success | bool
delegate_to: localhost
ansible.builtin.set_fact:
fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}"
- name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801')
when:
- indoor_success | bool
- (fw_banner_raw | default('') | length) > 0
delegate_to: localhost
ansible.builtin.set_fact:
fw_norm: >-
{{
fw_banner_raw
if (fw_banner_raw | lower is search('-r[0-9]+$'))
else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))
}}
- name: Debug normalized firmware (indoor_fwver)
when: fw_norm is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')"
- name: Publish custom_field_set indoor_fwver
when:
- indoor_success | bool
- fw_norm is defined
- (fw_norm | length) > 0
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ {
'inscope_device': (ansible_hostname | default(inventory_hostname)),
'task_name': 'custom_field_set',
'task_add1': 'indoor_fwver',
'task_result': fw_norm
} | to_json }}"
payload_encoding: "string"
register: rmq_customfield_fw
changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool)
# FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart
- name: Build failure journal payload (indoor firmware read failed)
when: not (indoor_success | bool)
delegate_to: localhost
ansible.builtin.set_fact:
journal_fail_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}):
fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}'
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish failure journal to control queue
when: journal_fail_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_fail_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_fail
changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool)
- name: Compute retry parameters (10 minutes)
when: not (indoor_success | bool)
ansible.builtin.set_fact:
next_attempt: "{{ (attempt | int) + 1 }}"
next_delay_sec: 600
next_delay_ms: "{{ 600000 }}"
- name: Build final gave-up journal (max attempts reached)
when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int))
delegate_to: localhost
ansible.builtin.set_fact:
journal_gaveup_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}.
Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }}
- name: Publish final gave-up journal
when: journal_gaveup_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_gaveup_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_gaveup
changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool)
# Only schedule next attempt if we still have budget left
- name: Build delayed payload for next indoor attempt (10 min)
when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int))
delegate_to: localhost
ansible.builtin.set_fact:
delayed_payload:
task_name: "afterupgrade_indoor_check"
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
attempt: "{{ next_attempt | int }}"
max_attempts: "{{ effective_max_attempts | int }}"
correlation_id: "{{ correlation_id | default('') }}"
original_emitted_at: "{{ original_emitted_at | default('') }}"
current_delay_sec: "{{ next_delay_sec | int }}"
schema_version: 1
- name: Publish delayed next indoor attempt (holding + TTL → deviceconfig)
when: delayed_payload is defined
delegate_to: localhost
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
expiration: "{{ (next_delay_ms | int) | string }}"
correlation_id: "{{ correlation_id | default('') }}"
routing_key: "deviceconfig"
payload: "{{ delayed_payload | to_json }}"
payload_encoding: "string"
register: rmq_pub_next
changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)
post_tasks:
- name: Cleanup tunnel and temp IP
block:
- ansible.builtin.debug:
msg: "Cleaning up tunnel + temp IP"
changed_when: false
delegate_to: localhost
always:
- name: Close tunnel
delegate_to: localhost
ansible.builtin.shell: |
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
changed_when: false
ignore_errors: true
- name: Remove control dir
delegate_to: localhost
ansible.builtin.file:
path: "{{ _ctrl_dir | default('/tmp/none') }}"
state: absent
ignore_errors: true
- name: Remove temporary IP from DEV1
ansible.builtin.raw: >
{{ pathprefix }}
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
register: del_ip
failed_when: false
changed_when: false

997
files/ansible-playbooks/update-indoor.yml-bckp1
View File

@@ -1,997 +0,0 @@
---
# update-secondline.yml
- name: Second-line upgrade tunneled noninvasive engine
hosts: all
gather_facts: no
vars:
# Busybox-safe PATH prefix for all remote raw calls on DEV1
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
# DEV1 credentials (stable, like rebootin222)
dev1_user: "root"
dev1_pass: "wavewave"
# Tunnel target DEV2 behind DEV1
dev2_host: "192.168.1.1"
dev2_port: 22
# Temp IP we add to DEV1 so it can reach DEV2
dev2_side_ip: "192.168.1.11/24"
dev1_iface: "br-wan"
# DEV2 behind the tunnel
dev2_ssh_user: "root"
dev2_passfiles:
- "basicpass"
- "basicpass2"
# SSH options used from controller
ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15"
# Image to stage on DEV2 noninvasive we do not run update -w here
image_filename: "fox200-2.2.1-r6801.bin"
image_md5: "56b7211709de617e058b98d4204e2562"
# Optional SHA256; leave empty to skip SHA256 checks
image_sha256: ""
dev2_image_dir: "/tmp"
dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}"
# Reboot delay in minutes integer string if set we will schedule reboot on DEV2
rebootin: ""
# ---------------- RabbitMQ journaling (mirrors rebootin222 style) ----------------
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
pre_tasks:
# ------------------------------- Hostname sanity DEV1 -------------------------------
- name: Read DEV1 hostname from env busybox safe
ansible.builtin.raw: >
{{ pathprefix }}
(cat /proc/sys/kernel/hostname 2>/dev/null || echo "")
register: dev1_host_read
changed_when: false
- name: Stop if DEV1 connected hostname differs from inventory
ansible.builtin.meta: end_host
when: (dev1_host_read.stdout | trim | length > 0) and
((dev1_host_read.stdout | trim) != (inventory_hostname | string))
tasks:
# ---------------------------- Idempotent temp IP on DEV1 ----------------------------
- name: Add temporary IP on DEV1 idempotent treat File exists as OK
ansible.builtin.raw: >
{{ pathprefix }}
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
register: add_ip
changed_when: add_ip.rc == 0
failed_when: >
add_ip.rc != 0
and ('File exists' not in (add_ip.stdout | default('')))
and ('File exists' not in (add_ip.stderr | default('')))
- name: Debug add ip results
ansible.builtin.debug:
msg:
- "add_ip.rc={{ add_ip.rc | default('') }}"
- "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
- "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
# ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ----------------------------
- name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb
ansible.builtin.raw: >
{{ pathprefix }}
bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
register: dev2_mac_scan
changed_when: false
- name: Set dev2_mac from fdb scan (if any)
ansible.builtin.set_fact:
dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
changed_when: false
- name: Remove current ARP entry for DEV2 on DEV1 (best-effort)
ansible.builtin.raw: >
{{ pathprefix }}
ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
register: dev2_arp_del
changed_when: false
failed_when: false
- name: Add static ARP for DEV2 on DEV1
ansible.builtin.raw: >
{{ pathprefix }}
ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
register: dev2_arp_add
changed_when: dev2_arp_add.rc == 0
failed_when: >
dev2_arp_add.rc != 0
and ('File exists' not in (dev2_arp_add.stdout | default('')))
and ('File exists' not in (dev2_arp_add.stderr | default('')))
- name: Debug static ARP result
ansible.builtin.debug:
msg:
- "dev2_mac={{ dev2_mac | default('UNSET') }}"
- "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
- "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
- "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
- name: Skip ARP add because MAC not found
when: dev2_mac is not defined or dev2_mac | length == 0
ansible.builtin.debug:
msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
# ---------------------------- Local tunnel preparation ----------------------------
- name: Pick a free local port for the tunnel controller
delegate_to: localhost
ansible.builtin.shell: |
set -e
pick() {
for i in $(seq 1 25); do
p="$(shuf -i 20000-39999 -n 1)"
if command -v ss >/dev/null 2>&1; then
if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then
echo "$p"; return 0
fi
else
if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then
echo "$p"; return 0
fi
fi
done
return 1
}
pick
register: pick_port
changed_when: false
- name: Stop if no free local port
ansible.builtin.meta: end_host
when: (pick_port.stdout | trim | length) == 0
- name: Record chosen local port and control directory
delegate_to: localhost
ansible.builtin.set_fact:
_local_port: "{{ pick_port.stdout | trim }}"
_ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}"
- name: Build tunnel control socket path
delegate_to: localhost
ansible.builtin.set_fact:
_ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl"
- name: Debug chosen tunnel params
delegate_to: localhost
ansible.builtin.debug:
msg:
- "local_port={{ _local_port }}"
- "ctrl_dir={{ _ctrl_dir }}"
- "ctrl_sock={{ _ctrl_sock }}"
- name: issue arping with own IP
ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3
# ---------------------------- Start SSH local forward via DEV1 ----------------------------
- name: Start SSH tunnel via DEV1 background ControlMaster
delegate_to: localhost
ansible.builtin.shell: |
set -e
USER="{{ dev1_user }}"
HOST="{{ ansible_host | default(inventory_hostname) }}"
sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
-M -S "{{ _ctrl_sock }}" \
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
"${USER}@${HOST}"
args:
executable: /bin/bash
register: start_tunnel
changed_when: true
- name: Small delay for tunnel to settle
delegate_to: localhost
ansible.builtin.wait_for:
timeout: 1
changed_when: false
- name: Verify tunnel master running proper O check with destination
delegate_to: localhost
ansible.builtin.shell: |
set -e
HOST="{{ ansible_host | default(inventory_hostname) }}"
ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true
register: tun_check
changed_when: false
- name: Debug tunnel check
delegate_to: localhost
ansible.builtin.debug:
msg:
- "tunnel_check.rc={{ tun_check.rc }}"
- "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}"
# ---------------------------- Controller-side sanity for DEV2 auth ----------------------------
- name: Sanity confirm tunnel TCP reachability to DEV2
delegate_to: localhost
ansible.builtin.shell: |
set -e
nc -z -w5 127.0.0.1 "{{ _local_port }}"
register: nc_probe
changed_when: false
ignore_errors: true
- name: Debug reachability result
delegate_to: localhost
ansible.builtin.debug:
msg:
- "nc.rc={{ nc_probe.rc }}"
- "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}"
- "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}"
- name: Stop if tunnel TCP check failed
ansible.builtin.meta: end_host
when: nc_probe.rc != 0
- name: Sanity show presence and permissions of passfiles on controller
delegate_to: localhost
ansible.builtin.shell: |
set -e
ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD"
register: dev2_ls
changed_when: false
- name: Debug passfiles listing
delegate_to: localhost
ansible.builtin.debug:
msg:
- "{{ (dev2_ls.stdout | default('')) | trim }}"
- "{{ (dev2_ls.stderr | default('')) | trim }}"
# ---------------------------- Pick DEV2 password root only ----------------------------
- name: Try DEV2 login with basicpass root
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f basicpass ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
register: dev2_try_basicpass
changed_when: false
ignore_errors: true
- name: Select basicpass as working passfile
when: dev2_try_basicpass.rc == 0
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "basicpass"
changed_when: false
- name: Try DEV2 login with basicpass2 root only if first failed
when: dev2_passfile_used is not defined
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f basicpass2 ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1
register: dev2_try_basicpass2
changed_when: false
ignore_errors: true
- name: Select basicpass2 as working passfile
when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "basicpass2"
changed_when: false
- name: Set dev2_passfile_used to NONE if neither worked
when: dev2_passfile_used is not defined
delegate_to: localhost
ansible.builtin.set_fact:
dev2_passfile_used: "NONE"
changed_when: false
- name: Debug selected DEV2 passfile or NONE
delegate_to: localhost
ansible.builtin.debug:
msg:
- "dev2_passfile_used={{ dev2_passfile_used }}"
- "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}"
- "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}"
- name: Read DEV2 hostname via tunnel (busybox-safe)
when: dev2_passfile_used != "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo"
args:
executable: /bin/bash
register: dev2_host_read
changed_when: false
- name: Normalize hostnames for strict compare
ansible.builtin.set_fact:
_inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}"
_dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
_dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}"
- name: Debug hostname bytes (hex)
delegate_to: localhost
ansible.builtin.debug:
msg:
- "inv={{ _inv_hn | tojson }}"
- "dev1={{ _dev1_hn | tojson }}"
- "dev2={{ _dev2_hn | tojson }}"
# ---------------- Hostname equality guard (soft-journal and stop) ----------------
- name: Enforce DEV2 hostname equals inventory and DEV1
block:
- name: Fail if DEV2 hostname differs from inventory/DEV1
ansible.builtin.fail:
msg: >
Hostname mismatch: DEV2='{{ _dev2_hn }}',
inventory='{{ _inv_hn }}',
DEV1='{{ _dev1_hn }}'
when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn)
rescue:
- name: Initialize journal array for hostname mismatch
ansible.builtin.set_fact:
_journal: []
_blocked: true
_prep_blocked: false
delegate_to: localhost
- name: Append hostname mismatch to journal
ansible.builtin.set_fact:
_journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}"
delegate_to: localhost
- name: Build control queue payload for indoor aborted journal (hostname)
ansible.builtin.set_fact:
journal_indoor_aborted:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
delegate_to: localhost
- name: Publish indoor aborted journal to control queue (hostname)
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_indoor_aborted | to_json }}"
payload_encoding: "string"
register: rmq_journal_indoor_aborted_hn_resp
changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)
failed_when: >
(rmq_journal_indoor_aborted_hn_resp.status != 200) or
(rmq_journal_indoor_aborted_hn_resp.json is not defined) or
(not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool))
delegate_to: localhost
- name: Stop host after hostname mismatch
ansible.builtin.meta: end_host
# ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ----------------------------
- name: Initialize journal and flags
ansible.builtin.set_fact:
_journal: []
_prep_blocked: false
_blocked: false
- name: Build specific marker filename (no .bin)
ansible.builtin.set_fact:
_marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}"
- name: Check for any preparation markers on DEV2 (wildcard count)
when: dev2_passfile_used != "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"ls /tmp/prepared_for* 2>/dev/null | wc -l"
args:
executable: /bin/bash
register: dev2_prep_count
changed_when: false
ignore_errors: true
- name: Soft-block if any prep markers already present
when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0)
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
# ---------------------------- DEV2 version firmux primary check ----------------------------
- name: Read DEV2 firmux release
when: dev2_passfile_used != "NONE"
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"cat /usr/lib/release/firmux 2>/dev/null || true"
args:
executable: /bin/bash
register: dev2_firmux
changed_when: false
ignore_errors: true
- name: Debug firmux content DEV2
when: dev2_firmux is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
# ---------------------------- Normalize rebootin early ----------------------------
- name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)"
delegate_to: localhost
ansible.builtin.set_fact:
_reboot_raw: "{{ rebootin | default('') | string | trim | lower }}"
_reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}"
_reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}"
_reboot_minutes: >-
{{
0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'])
else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1)
}}
- name: "Normalize rebootin (phase 2: requested flag)"
delegate_to: localhost
ansible.builtin.set_fact:
_reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}"
# ---------------------------- Journal: indoor start (after we know we can proceed) ----------------------------
- name: Build control queue payload for indoor start journal
ansible.builtin.set_fact:
journal_indoor_start:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
Indoor: Dev2 is reachable, starting with update.
image={{ image_filename }}, md5={{ image_md5 }},
firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }},
reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }}
when: dev2_passfile_used != "NONE"
delegate_to: localhost
- name: Publish indoor start journal to control queue
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_indoor_start | to_json }}"
payload_encoding: "string"
register: rmq_journal_indoor_start_resp
changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)
failed_when: >
(rmq_journal_indoor_start_resp.status != 200) or
(rmq_journal_indoor_start_resp.json is not defined) or
(not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool))
when: journal_indoor_start is defined
delegate_to: localhost
# ---------------------------- Stage image on DEV2 no write ----------------------------
- name: Check local presence of image file
delegate_to: localhost
ansible.builtin.stat:
path: "{{ image_filename }}"
register: local_img
- name: Soft-block if image missing locally
when: not local_img.stat.exists
ansible.builtin.set_fact:
_blocked: true
_journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}"
- name: Compute local md5 of the image
when: local_img.stat.exists
delegate_to: localhost
ansible.builtin.shell: |
set -e
md5sum "{{ image_filename }}" | awk '{print $1}'
register: local_md5
changed_when: false
- name: Verify local md5 matches expected
when: local_img.stat.exists
delegate_to: localhost
ansible.builtin.assert:
that:
- (local_md5.stdout | trim) == image_md5
fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}"
success_msg: "Local md5 matches expected"
# Optional SHA256 local
- name: Compute local sha256 of the image (if provided)
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
sha256sum "{{ image_filename }}" | awk '{print $1}'
register: local_sha256
changed_when: false
ignore_errors: true
- name: Soft-block if local sha256 mismatch or unavailable
when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim))
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
# fw_printenv health before upload (soft-fail)
- name: Read fw_printenv size (line count) on DEV2
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"(fw_printenv 2>/dev/null | wc -l) || echo 0"
args:
executable: /bin/bash
register: dev2_fwenv_wc
changed_when: false
ignore_errors: true
- name: Soft-block if fw_printenv too small
when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30)
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}"
- name: Check existing image md5 on DEV2 NOFILE if absent
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE"
args:
executable: /bin/bash
register: dev2_md5_before
changed_when: false
- name: Copy image to DEV2 via tunnel only if missing or md5 mismatch
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5)
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" scp \
-P "$PORT" \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
"{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/"
args:
executable: /bin/bash
register: scp_push
changed_when: true
- name: Compute md5 of image on DEV2 after copy or if exists
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE"
args:
executable: /bin/bash
register: dev2_md5_after
changed_when: false
- name: Verify DEV2 md5 matches expected (soft-fail journal)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5)
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}"
# Optional SHA256 remote
- name: Compute sha256 of image on DEV2 (if provided)
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA"
args:
executable: /bin/bash
register: dev2_sha256_after
changed_when: false
ignore_errors: true
- name: Soft-block if DEV2 sha256 mismatch/unavailable
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim))
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}"
- name: Run noninvasive update check on DEV2 update c
when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=15 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"update -c '{{ dev2_image_path }}' 2>&1 || true"
args:
executable: /bin/bash
register: dev2_update_check
changed_when: false
- name: Debug update c output DEV2
when: dev2_update_check is defined
delegate_to: localhost
ansible.builtin.debug:
msg: "{{ (dev2_update_check.stdout | default('')) | trim }}"
- name: Soft-block if update -c did not say valid
when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid'))
ansible.builtin.set_fact:
_prep_blocked: true
_blocked: true
_journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}"
# ---------------------------- Journal: indoor aborted (if any blockers) ----------------------------
- name: Build control queue payload for indoor aborted journal
ansible.builtin.set_fact:
journal_indoor_aborted:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }}
when: (_blocked | default(false))
delegate_to: localhost
- name: Publish indoor aborted journal to control queue
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_indoor_aborted | to_json }}"
payload_encoding: "string"
register: rmq_journal_indoor_aborted_resp
changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)
failed_when: >
(rmq_journal_indoor_aborted_resp.status != 200) or
(rmq_journal_indoor_aborted_resp.json is not defined) or
(not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool))
when: journal_indoor_aborted is defined
delegate_to: localhost
# ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================
- name: Upgrade write and flip block
when: not (_blocked | default(false))
block:
- name: Write image on DEV2 (this will take a while)
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=0 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"update -w '{{ dev2_image_path }}'"
args:
executable: /bin/bash
register: dev2_up_write
changed_when: true
failed_when: dev2_up_write.stdout is not search('update is complete')
- name: Read current active partition on DEV2
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"fw_printenv active | awk -F= '/^active=/{print \$2}'"
args:
executable: /bin/bash
register: dev2_active_before
changed_when: false
failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
- name: Determine new active value for DEV2
when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
ansible.builtin.set_fact:
dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
- name: Switch active partition to {{ dev2_new_active }}
when: dev2_new_active is defined
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"fw_setenv active {{ dev2_new_active }}"
args:
executable: /bin/bash
register: dev2_setenv_out
changed_when: true
- name: Verify active partition flipped on DEV2
when: dev2_new_active is defined
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"fw_printenv active | awk -F= '/^active=/{print \$2}'"
args:
executable: /bin/bash
register: dev2_active_after
changed_when: false
failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
- name: Create specific prep marker on DEV2 (no generic)
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
"touch '{{ _marker_specific }}'"
args:
executable: /bin/bash
register: dev2_marker_write
changed_when: true
ignore_errors: true
# ---------------------------- Reboot scheduling (normalized) ----------------------------
- name: Schedule delayed reboot on DEV2 minutes
when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))
delegate_to: localhost
ansible.builtin.shell: |
set -e
PORT="{{ _local_port }}"
SECS="$(( {{ _reboot_minutes | int }} * 60 ))"
CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &'
sshpass -f "{{ dev2_passfile_used }}" ssh \
-o AddressFamily=inet \
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-o ConnectTimeout=10 \
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}"
args:
executable: /bin/bash
register: dev2_reboot_sched
changed_when: true
ignore_errors: true
- name: Note reboot request bad format
when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested
ansible.builtin.debug:
msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied"
# ---------------------------- Journal: indoor updated and reboot schedule ----------------------------
- name: Build control queue payload for indoor updated journal
ansible.builtin.set_fact:
_write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}"
when: not (_blocked | default(false))
delegate_to: localhost
- name: Build indoor updated journal payload text
ansible.builtin.set_fact:
journal_indoor_updated:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
Indoor: Dev2 updated, scheduling reboot {{
'now' if (_reboot_requested and (_reboot_minutes | int) == 0)
else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0)
else 'not requested'
}}.
write_done={{ _write_success }},
active_before={{ (dev2_active_before.stdout | default('NA')) | trim }},
active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}
when: not (_blocked | default(false)) and (_write_success | bool)
delegate_to: localhost
- name: Publish indoor updated journal to control queue
ansible.builtin.uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ journal_indoor_updated | to_json }}"
payload_encoding: "string"
register: rmq_journal_indoor_updated_resp
changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)
failed_when: >
(rmq_journal_indoor_updated_resp.status != 200) or
(rmq_journal_indoor_updated_resp.json is not defined) or
(not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool))
when: journal_indoor_updated is defined
delegate_to: localhost
# ---------------------------- summary ----------------------------
- name: Summary show key results
delegate_to: localhost
ansible.builtin.debug:
msg:
- "dev2_passfile_used={{ dev2_passfile_used }}"
- "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}"
- "local_image_present={{ local_img.stat.exists | default(false) }}"
- "local_md5={{ (local_md5.stdout | default('NA')) | trim }}"
- "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}"
- "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}"
- "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}"
- "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}"
- "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}"
- "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}"
- "reboot_requested={{ _reboot_requested | default(false) }}"
- "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}"
- "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}"
- "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}"
- "prep_blocked={{ _prep_blocked | default(false) }}"
- "blocked={{ _blocked | default(false) }}"
- "journal={{ (_journal | default([])) | join(' || ') }}"
post_tasks:
- name: Cleanup always
block:
- ansible.builtin.debug:
msg: "entering cleanup block"
changed_when: false
delegate_to: localhost
always:
- name: Close tunnel best effort
delegate_to: localhost
ansible.builtin.shell: |
ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true
changed_when: false
ignore_errors: true
- name: Remove control dir best effort
delegate_to: localhost
ansible.builtin.file:
path: "{{ _ctrl_dir | default('/tmp/none') }}"
state: absent
ignore_errors: true
- name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK
ansible.builtin.raw: >
{{ pathprefix }}
ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}
register: del_ip
changed_when: del_ip.rc == 0
failed_when: >
del_ip.rc != 0
and ('Cannot assign requested address' not in (del_ip.stdout | default('')))
and ('Cannot assign requested address' not in (del_ip.stderr | default('')))
- name: Debug del ip results
ansible.builtin.debug:
msg:
- "del_ip.rc={{ del_ip.rc | default('') }}"
- "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}"
- "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}"
when: del_ip is defined

1108
files/ansible-playbooks/update-indoor.yml-beforefixrebootin
View File

File diff suppressed because it is too large Load Diff

1100
files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix
View File

File diff suppressed because it is too large Load Diff

1117
files/ansible-playbooks/update-indoor.yml-day2
View File

File diff suppressed because it is too large Load Diff

1117
files/ansible-playbooks/update-indoor.yml-fixingerrors
View File

File diff suppressed because it is too large Load Diff

1108
files/ansible-playbooks/update-indoor.yml2
View File

File diff suppressed because it is too large Load Diff

441
files/ansible-playbooks/wifidebug17.yml-old
View File

@@ -1,441 +0,0 @@
---
- name: Deploy WiFi debug (single device, linear)
hosts: all
gather_facts: no
vars:
remote_syslog_ip: "102.38.125.161"
# Program version (0)
wdbg_version: "v17"
ssh_user: "{{ ansible_user | default('root') }}"
ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}"
# RabbitMQ (use controls exchange + queue_controls like the reference) (e)
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}"
tasks:
- block:
# --- SSH reachability check ---
- name: Check SSH connectivity (raw ping)
raw: "echo ping"
register: ping_result
ignore_errors: true
- block:
############ step 1
- name: Check if crontab launch is already present in sysstart.lua
raw: |
grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT
register: sysstart_cron_check
changed_when: false
failed_when: false
- name: Announce presence and skip edits
when: (sysstart_cron_check.stdout | trim) == 'PRESENT'
debug:
msg: "crontab is present in sysstart.lua; skipping download/edit/upload."
- name: Fetch sysstart.lua via scp (password auth)
when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
delegate_to: localhost
command: >
sshpass -p {{ ssh_pass | quote }}
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
{{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
/tmp/{{ inventory_hostname }}_sysstart.lua
register: scp_get
retries: 3
delay: 2
until: scp_get.rc == 0
- name: Edit the file locally
when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
delegate_to: localhost
lineinfile:
path: "/tmp/{{ inventory_hostname }}_sysstart.lua"
line: 'launchd.set_process("crond", "/usr/sbin/crond -f")'
insertbefore: '^tasks\.start\(ctx,\s*"finished"\)'
- name: Push the file back (write to -2)
when: (sysstart_cron_check.stdout | trim) != 'PRESENT'
delegate_to: localhost
command: >
sshpass -p {{ ssh_pass | quote }}
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
/tmp/{{ inventory_hostname }}_sysstart.lua
{{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua
register: scp_put
retries: 3
delay: 2
until: scp_put.rc == 0
############ step 2
- name: Compute MD5 of local wifidebug.sh
delegate_to: localhost
command: md5sum files/wifidebug.sh
register: md5_local_wdbg
changed_when: false
- name: Compute MD5 of remote /root/wifidebug.sh
raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh"
register: md5_remote_wdbg
changed_when: false
failed_when: false
- name: Decide if wifidebug.sh needs upload
set_fact:
upload_wdbg: >-
{{ (md5_remote_wdbg.rc != 0)
or ((md5_local_wdbg.stdout.split()[0])
!= (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }}
- name: Upload wifidebug.sh via scp (overwrite if changed)
when: upload_wdbg | bool
delegate_to: localhost
command: >
sshpass -p {{ ssh_pass | quote }}
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
files/wifidebug.sh
{{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh
register: scp_wifidebug
retries: 3
delay: 2
until: scp_wifidebug.rc == 0
- name: Ensure /root/wifidebug.sh is executable and owned by root
raw: |
chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh
############ step 3
- name: Ensure /etc/crontabs/root exists (touch with perms)
raw: |
if [ ! -f /etc/crontabs/root ]; then
touch /etc/crontabs/root
fi
chown root:root /etc/crontabs/root
chmod 0644 /etc/crontabs/root
- name: Check if cron line already present
raw: |
grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root
register: cron_grep
failed_when: false
changed_when: false
- name: Upload snippet crond-root to /tmp (only if missing)
when: cron_grep.rc != 0
delegate_to: localhost
command: >
sshpass -p {{ ssh_pass | quote }}
scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
files/crond-root
{{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet
- name: Append snippet to /etc/crontabs/root (only if missing)
when: cron_grep.rc != 0
raw: |
cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet
register: cron_append
changed_when: true
############ step 4
- name: Skip config.json edit if remote_syslog already matches
raw: |
EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0)
SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0)
if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then
echo "SKIP"
else
echo "EDIT"
fi
register: remote_syslog_check
changed_when: false
- name: Set do_edit flag from skip check
set_fact:
do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}"
- name: Fetch /tmp/config.json to controller
when: do_edit
delegate_to: localhost
command: >
sshpass -p {{ ssh_pass | quote }}
scp -q -o StrictHostKeyChecking=no
{{ ssh_user }}@{{ ansible_host }}:/tmp/config.json
/tmp/{{ inventory_hostname }}_config.json
- name: Strict in-place style-preserving edit (services.remote_syslog only)
when: do_edit
delegate_to: localhost
shell: |
awk '
BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 }
{
line = $0
if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 }
if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 }
if (in_rs) {
if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) {
line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line)
done_enabled=1
}
if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) {
line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line)
done_server=1
}
}
print line
if (in_rs) {
opens = gsub(/{/, "{", $0)
closes = gsub(/}/, "}", $0)
depth += (opens - closes)
if (depth <= 0 && $0 ~ /}/) { in_rs=0 }
}
}
' "/tmp/{{ inventory_hostname }}_config.json" \
> "/tmp/{{ inventory_hostname }}_config.json.new"
args:
executable: /bin/bash
- name: Push edited config.json.new to remote temp
when: do_edit
delegate_to: localhost
shell: |
sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \
"/tmp/{{ inventory_hostname }}_config.json.new" \
{{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new
args:
executable: /bin/bash
- name: Compute MD5 of local .new
when: do_edit
delegate_to: localhost
command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new"
register: md5_local
changed_when: false
- name: Compute MD5 of remote .new (no Python)
when: do_edit
raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new"
register: md5_remote
changed_when: false
- name: Fail if MD5 mismatch
when:
- do_edit
- (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD')
fail:
msg: "MD5 mismatch between controller and remote copy — aborting replace."
- name: Commit new config.json (overwrite original)
when: do_edit
raw: |
cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S)
mv /tmp/config.json.new /tmp/config.json
- name: Apply config to runtime (sysconf -w)
when: do_edit
raw: |
sysconf -w || /usr/sbin/sysconf -w
register: sysconf_result
changed_when: true
failed_when: sysconf_result.rc not in [0]
# (d) Pre-restart notice to control queue
- name: Build pre-restart journal (explain why)
when: do_edit
delegate_to: localhost
set_fact:
prereboot_payload:
inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
task_name: "journal_add"
task_result: >-
wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w).
- name: Publish pre-restart journal to control queue
when: do_edit
delegate_to: localhost
uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ prereboot_payload | to_json }}"
payload_encoding: "string"
register: rmq_preboot
changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool)
# ======== SURGICAL CHANGE: restart kick + verification like multissidfix ========
- name: Trigger full restart (system-stop; system-start) from controller with 10s cap
when: do_edit
delegate_to: localhost
shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'"
args:
executable: /bin/bash
register: restart_kick
changed_when: true
failed_when: false
# probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3)
- name: Probe SSH with nc (24 tries, 5s each)
when: do_edit
delegate_to: localhost
shell: "nc -z -w 3 {{ ansible_host }} 22"
register: nc_probe
retries: 24
delay: 5
until: nc_probe.rc == 0
changed_when: false
failed_when: false
- name: Set ssh_up fact (from nc probe)
when: do_edit
delegate_to: localhost
set_fact:
ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}"
- name: Fail if SSH did not return after restart
when:
- do_edit
- not ssh_up | bool
fail:
msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks."
# ======== END OF SURGICAL CHANGE ========
- name: Set result status (success deployed or no change)
set_fact:
result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}"
when: ping_result is succeeded
- name: Set status fact (no ssh)
when: ping_result is failed
set_fact:
result_status: "NO_SSH"
rescue:
- name: Mark result as failed
set_fact:
result_status: "FAILED during {{ ansible_failed_task.name }}"
always:
- name: Compute inscope device
set_fact:
inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}"
# (a) Set custom field wifidebug -> v15 on control queue
- name: Build custom-field payload (wifidebug -> version)
delegate_to: localhost
set_fact:
wdbg_cf_payload:
inscope_device: "{{ inscope_device_name }}"
task_name: "deploy_wifidebug"
task_result: "{{ wdbg_version }}"
- name: Publish custom-field update to control queue
delegate_to: localhost
uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ wdbg_cf_payload | to_json }}"
payload_encoding: "string"
register: rmq_cf
changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool)
# (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed
- name: Build actions list
set_fact:
_actions_list: >-
{{
[]
+ ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], []))
+ (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], []))
+ ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], []))
+ ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], []))
+ ((do_edit | default(false) | bool) | ternary(['restarted services'], []))
}}
- name: Build actions string
set_fact:
_actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}"
- name: Build wrap-up journal payload
delegate_to: localhost
set_fact:
wrap_payload:
inscope_device: "{{ inscope_device_name }}"
task_name: "journal_add"
task_result: >-
wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }}
— actions: {{ _actions_str }}
- name: Publish wrap-up journal to control queue
delegate_to: localhost
uri:
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
method: POST
user: "{{ rmq_user }}"
password: "{{ rmq_pass }}"
force_basic_auth: true
status_code: 200
headers:
content-type: "application/json"
body_format: json
body:
properties:
content_type: "application/json"
routing_key: "{{ control_queue }}"
payload: "{{ wrap_payload | to_json }}"
payload_encoding: "string"
register: rmq_wrap
changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool)
# Local summary (kept for operator visibility)
- name: Summary
debug:
msg:
- "result_status: {{ result_status }}"
- "we're good"