502 lines
18 KiB
YAML
502 lines
18 KiB
YAML
---
|
|
# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode)
|
|
|
|
- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry)
|
|
hosts: all
|
|
gather_facts: no
|
|
|
|
vars:
|
|
pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
|
|
dev1_user: "root"
|
|
dev1_pass: "wavewave"
|
|
ssh_timeout: 30
|
|
|
|
dev2_host: "192.168.1.1"
|
|
dev2_port: 22
|
|
dev2_ssh_user: "root"
|
|
dev2_passfiles: [ "basicpass", "basicpass2" ]
|
|
|
|
dev2_side_ip: "192.168.1.11/24"
|
|
dev1_iface: "br-wan"
|
|
arping_iface: "eth0"
|
|
dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"
|
|
|
|
ssh_opts_common: >-
|
|
-o PreferredAuthentications=password
|
|
-o PubkeyAuthentication=no
|
|
-o StrictHostKeyChecking=no
|
|
-o UserKnownHostsFile=/dev/null
|
|
-o NumberOfPasswordPrompts=1
|
|
-o ConnectTimeout=30
|
|
|
|
rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
|
|
rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
|
|
rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
|
|
rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
|
|
rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
|
|
rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
|
|
control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
|
|
|
|
pre_tasks:
|
|
- name: "NB preflight | Verify script exists"
|
|
delegate_to: localhost
|
|
ansible.builtin.stat:
|
|
path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
|
|
register: nb_script
|
|
|
|
- name: "NB preflight | Abort softly if script missing (path typo?)"
|
|
when: not nb_script.stat.exists
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
|
|
- "Tip: adjust chdir/path or script name."
|
|
|
|
- name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }} (chatty)"
|
|
when: nb_script.stat.exists
|
|
delegate_to: localhost
|
|
environment:
|
|
PYTHONUNBUFFERED: "1"
|
|
args:
|
|
chdir: "/opt/containers/ansible-worker/app"
|
|
executable: /bin/bash
|
|
shell: |
|
|
set -o pipefail
|
|
python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
|
|
register: nb_preflight
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: "NB preflight | Show results"
|
|
when: nb_script.stat.exists
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "rc={{ nb_preflight.rc }}"
|
|
- "stdout_lines:"
|
|
- "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
|
|
- "stderr_lines:"
|
|
- "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
|
|
- "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"
|
|
|
|
# --- Parse the OK line for reuse (device/ip/fw/etc.) ---
|
|
- name: "NB preflight | Parse OK line"
|
|
when: nb_script.stat.exists
|
|
delegate_to: localhost
|
|
vars:
|
|
nb_ok_line: >-
|
|
{{ (nb_preflight.stdout | default('') | regex_findall('^OK\\s+.*$', multiline=True)) | last | default('') }}
|
|
nb_groups: >-
|
|
{{ (nb_ok_line | regex_findall('^OK\\s+(\\S+)\\s+ip=([^\\s]+)\\s+fw=([^\\s]+)\\s+node=([^\\s]+)\\s+sector=([^\\s]+)\\s+small=([^\\s]+)')) | first | default([]) }}
|
|
set_fact:
|
|
nb_ok: "{{ (nb_preflight.rc | default(1)) == 0 and (nb_groups | length) == 6 }}"
|
|
nb_dev: "{{ nb_groups[0] | default('') }}"
|
|
nb_ip: "{{ nb_groups[1] | default('') }}"
|
|
nb_fw: "{{ nb_groups[2] | default('') }}"
|
|
nb_node: "{{ nb_groups[3] | default('') }}"
|
|
nb_sector: "{{ nb_groups[4] | default('') }}"
|
|
nb_small: "{{ nb_groups[5] | default('') }}"
|
|
nb_ok_line: "{{ nb_ok_line }}"
|
|
|
|
# --- Heuristic for "cloud IP different from NetBox before update" ---
|
|
# We detect this via the script's chatty line: "IP: moving <old-ip> from iface=... -> ..."
|
|
- name: "NB preflight | Detect whether IP changed (cloud vs prior NetBox)"
|
|
when: nb_script.stat.exists
|
|
delegate_to: localhost
|
|
set_fact:
|
|
nb_ip_changed: "{{ (nb_preflight.stdout | default('')) is search('^IP:\\s+moving\\s+[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\s+from\\s+iface=', multiline=True) }}"
|
|
|
|
- name: "NB preflight | Verdict"
|
|
when: nb_script.stat.exists
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "NB preflight verdict: {{ 'IP CHANGED (will requeue)' if nb_ip_changed else 'IP SAME (continue)' }}"
|
|
- "OK line: {{ nb_ok_line | default('<none>') }}"
|
|
- "Parsed nb_ip={{ nb_ip | default('<n/a>') }}"
|
|
|
|
# --- If IP changed → publish a 3s delayed 'sot-updater' task and stop this host ---
|
|
- name: "NB preflight | Publish delayed requeue (3s) and stop"
|
|
when:
|
|
- nb_script.stat.exists
|
|
- nb_ip_changed | default(false)
|
|
delegate_to: localhost
|
|
uri:
|
|
url: "http://localhost:15672/api/exchanges/app/deviceconfig.delayed/publish"
|
|
method: POST
|
|
user: "admin"
|
|
password: "change_me"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers:
|
|
content-type: "application/json"
|
|
body_format: json
|
|
body:
|
|
properties:
|
|
content_type: "application/json"
|
|
headers:
|
|
x-delay: 3000
|
|
routing_key: "deviceconfig"
|
|
payload: "{{ {'inscope_device': (ansible_hostname | default(inventory_hostname)), 'task_name': 'sot-updater'} | to_json }}"
|
|
payload_encoding: "string"
|
|
register: rmq_requeue
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: "NB preflight | Log requeue publish response"
|
|
when:
|
|
- nb_script.stat.exists
|
|
- nb_ip_changed | default(false)
|
|
delegate_to: localhost
|
|
ansible.builtin.debug:
|
|
var: rmq_requeue.json
|
|
|
|
- name: "NB preflight | Stop further tasks for this host"
|
|
when:
|
|
- nb_script.stat.exists
|
|
- nb_ip_changed | default(false)
|
|
meta: end_host
|
|
|
|
# --- If IP did not change → optional 1s pause, then continue normally ---
|
|
- name: "NB preflight | Pause 1s"
|
|
when:
|
|
- nb_script.stat.exists
|
|
- not (nb_ip_changed | default(false))
|
|
delegate_to: localhost
|
|
ansible.builtin.pause:
|
|
seconds: 1
|
|
|
|
|
|
tasks:
|
|
- name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
|
|
debug:
|
|
msg:
|
|
- "Device: {{ inventory_hostname }}"
|
|
- "Mode: report-only (soft-fail; publish journals on failures)"
|
|
|
|
# ----------------------- Temp IP on DEV1 -----------------------
|
|
- name: Add temporary IP on DEV1
|
|
raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
|
|
register: add_ip
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
# ---------------------------- Idempotent temp IP on DEV1 ----------------------------
|
|
- name: Add temporary IP on DEV1 (tolerate 'File exists')
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
|
|
register: add_ip
|
|
changed_when: add_ip.rc == 0
|
|
failed_when: >
|
|
add_ip.rc != 0
|
|
and ('File exists' not in (add_ip.stdout | default('')))
|
|
and ('File exists' not in (add_ip.stderr | default('')))
|
|
|
|
- name: Debug result of adding temp IP to DEV1
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "add_ip.rc={{ add_ip.rc | default('') }}"
|
|
- "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}"
|
|
- "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}"
|
|
|
|
# ---------------------------- Discover MAC via bridge FDB and add static ARP ----------------------------
|
|
- name: Discover DEV2 MAC via bridge fdb on DEV1 (best-effort)
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
|
|
register: dev2_mac_scan
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: Capture discovered DEV2 MAC (if any)
|
|
ansible.builtin.set_fact:
|
|
dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}"
|
|
changed_when: false
|
|
|
|
- name: Clear existing ARP entry for DEV2 on DEV1 (best-effort)
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
|
|
register: dev2_arp_del
|
|
changed_when: false
|
|
failed_when: false
|
|
|
|
- name: Add static ARP entry on DEV1 (locks DEV2 IP → discovered MAC)
|
|
when: dev2_mac is defined and dev2_mac | length > 0
|
|
ansible.builtin.raw: >
|
|
{{ pathprefix }}
|
|
ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
|
|
register: dev2_arp_add
|
|
changed_when: dev2_arp_add.rc == 0
|
|
failed_when: >
|
|
dev2_arp_add.rc != 0
|
|
and ('File exists' not in (dev2_arp_add.stdout | default('')))
|
|
and ('File exists' not in (dev2_arp_add.stderr | default('')))
|
|
|
|
- name: Debug ARP action summary on DEV1
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "dev2_mac={{ dev2_mac | default('UNSET') }}"
|
|
- "arp_add.rc={{ dev2_arp_add.rc | default('') }}"
|
|
- "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}"
|
|
- "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}"
|
|
|
|
- name: Note skipping static ARP add (no MAC discovered)
|
|
when: dev2_mac is not defined or dev2_mac | length == 0
|
|
ansible.builtin.debug:
|
|
msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1"
|
|
|
|
# ---------------------------- ARP refresh ----------------------------
|
|
- name: Refresh ARP #1
|
|
ansible.builtin.raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
|
|
- name: Refresh ARP #1
|
|
raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
# ----------------------- Local tunnel prep -----------------------
|
|
- name: Pick a free local TCP port for the tunnel
|
|
delegate_to: localhost
|
|
shell: |
|
|
for i in $(seq 1 50); do
|
|
p="$(shuf -i 20000-39999 -n 1)"
|
|
ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
|
|
done
|
|
exit 1
|
|
register: pick_port
|
|
failed_when: false
|
|
|
|
- name: Record chosen port and create control dir
|
|
delegate_to: localhost
|
|
shell: "mktemp -d"
|
|
register: mktemp_dir
|
|
failed_when: false
|
|
|
|
- name: Set facts for tunnel paths
|
|
delegate_to: localhost
|
|
set_fact:
|
|
_local_port: "{{ (pick_port.stdout | default('') | trim) }}"
|
|
_ctrl_dir: "{{ (mktemp_dir.stdout | default('') | trim) }}"
|
|
_ctrl_sock: "{{ (mktemp_dir.stdout | default('') | trim) }}/ssh_tunnel_ctl"
|
|
|
|
# ----------------------- AIRPING #2 -----------------------
|
|
- name: Refresh ARP #2
|
|
raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
# ----------------------- Start tunnel -----------------------
|
|
- name: Start SSH tunnel via DEV1
|
|
delegate_to: localhost
|
|
shell: |
|
|
set -e
|
|
USER="{{ dev1_user }}"
|
|
HOST="{{ ansible_host | default(inventory_hostname) }}"
|
|
sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
|
|
-M -S "{{ _ctrl_sock }}" \
|
|
-L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
|
|
"${USER}@${HOST}"
|
|
args: { executable: /bin/bash }
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Probe tunnel
|
|
delegate_to: localhost
|
|
shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
|
|
register: nc_probe
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Set tunnel_ok
|
|
delegate_to: localhost
|
|
set_fact:
|
|
tunnel_ok: "{{ nc_probe.rc == 0 }}"
|
|
|
|
# ----------------------- DEV1 banner -----------------------
|
|
- name: Dev1 | Probe banner
|
|
delegate_to: localhost
|
|
shell: |
|
|
sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
|
|
"{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" \
|
|
"cat /etc/banner | grep -i rev | head -n1"
|
|
register: dev1_banner
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Dev1 | Normalize banner → X.X.X-rYYYY (POSIX tools)
|
|
delegate_to: localhost
|
|
shell: |
|
|
printf '%s\n' "{{ dev1_banner.stdout | trim }}" \
|
|
| awk -F '|' '{print $1}' \
|
|
| sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
|
|
| grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
|
|
register: dev1_fw_clean_cmd
|
|
changed_when: false
|
|
|
|
- name: Dev1 | Set final fw string
|
|
delegate_to: localhost
|
|
set_fact:
|
|
dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout | trim }}"
|
|
|
|
- name: Debug Dev1 normalized firmware
|
|
delegate_to: localhost
|
|
debug:
|
|
msg: "Dev1 fw_version {{ dev1_fw_clean | default('N/A') }}"
|
|
|
|
- name: Publish Dev1 fw_version
|
|
delegate_to: localhost
|
|
uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'fw_version',
|
|
'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
|
|
# ----------------------- DEV2 auth -----------------------
|
|
- name: Try DEV2 login
|
|
when: tunnel_ok | default(false)
|
|
delegate_to: localhost
|
|
shell: |
|
|
for f in basicpass basicpass2; do
|
|
PORT="{{ _local_port }}"
|
|
if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
-p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
|
|
echo "$f"; exit 0
|
|
fi
|
|
done
|
|
exit 1
|
|
register: dev2_auth
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Set dev2_passfile_used
|
|
delegate_to: localhost
|
|
set_fact:
|
|
dev2_passfile_used: "{{ (dev2_auth.rc == 0) | ternary(dev2_auth.stdout | trim, 'NONE') }}"
|
|
|
|
# ----------------------- AIRPING #3 -----------------------
|
|
- name: Refresh ARP #3
|
|
raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
|
|
# ----------------------- DEV2 firmux (simplified, literal) -----------------------
|
|
- name: Dev2 | Read /usr/lib/release/firmux
|
|
when: tunnel_ok | default(false) and dev2_passfile_used != 'NONE'
|
|
delegate_to: localhost
|
|
shell: |
|
|
PORT="{{ _local_port }}"
|
|
sshpass -f "{{ dev2_passfile_used }}" ssh \
|
|
-o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
|
|
-o ConnectTimeout=15 \
|
|
-p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
|
|
"cat /usr/lib/release/firmux 2>/dev/null || true"
|
|
register: dev2_firmux
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Dev2 | Extract firmware version line (prefer 'rev', else first non-empty)
|
|
delegate_to: localhost
|
|
run_once: true
|
|
set_fact:
|
|
indoor_fw_norm: >-
|
|
{{
|
|
(
|
|
(
|
|
dev2_firmux.stdout | default('') | regex_replace('\r','')
|
|
) | split('\n')
|
|
| map('trim')
|
|
| select('truthy')
|
|
| list
|
|
) | select('match', '(?i).*\\brev\\s*[0-9]+.*')
|
|
| list
|
|
| first
|
|
| default(
|
|
((dev2_firmux.stdout | default('') | regex_replace('\r',''))
|
|
| split('\n')
|
|
| map('trim')
|
|
| select('truthy')
|
|
| list
|
|
| first
|
|
| default(''))
|
|
)
|
|
| trim
|
|
}}
|
|
|
|
- name: Publish Dev2 indoor_fwver
|
|
delegate_to: localhost
|
|
uri:
|
|
url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
|
|
method: POST
|
|
user: "{{ rmq_user }}"
|
|
password: "{{ rmq_pass }}"
|
|
force_basic_auth: true
|
|
status_code: 200
|
|
headers: { content-type: "application/json" }
|
|
body_format: json
|
|
body:
|
|
properties: { content_type: "application/json" }
|
|
routing_key: "{{ control_queue }}"
|
|
payload: "{{ {
|
|
'inscope_device': (ansible_hostname | default(inventory_hostname)),
|
|
'task_name': 'custom_field_set',
|
|
'task_add1': 'indoor_fwver',
|
|
'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
|
|
} | to_json }}"
|
|
payload_encoding: "string"
|
|
changed_when: false
|
|
|
|
post_tasks:
|
|
- name: "✅ Cleanup start"
|
|
debug:
|
|
msg: "Closing tunnel and removing temporary IP"
|
|
|
|
- name: Refresh ARP #4
|
|
raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Close SSH ControlMaster
|
|
delegate_to: localhost
|
|
shell: "ssh -S '{{ _ctrl_sock | default('/dev/null') }}' -O exit 2>/dev/null || true"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: Remove tunnel control dir
|
|
delegate_to: localhost
|
|
file:
|
|
path: "{{ _ctrl_dir | default('/tmp/none') }}"
|
|
state: absent
|
|
ignore_errors: true
|
|
|
|
- name: Remove temporary IP
|
|
raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
|
|
failed_when: false
|
|
ignore_errors: true
|
|
|
|
- name: "✅ Completed | Device processed"
|
|
debug:
|
|
msg:
|
|
- "Device: {{ inventory_hostname }}"
|
|
- "Status: DONE"
|