--- # update-secondline.yml - name: Second-line upgrade tunneled noninvasive engine hosts: all gather_facts: no vars: # Busybox-safe PATH prefix for all remote raw calls on DEV1 pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " # DEV1 credentials (stable, like rebootin222) dev1_user: "root" dev1_pass: "wavewave" # Tunnel target DEV2 behind DEV1 dev2_host: "192.168.1.1" dev2_port: 22 # Temp IP we add to DEV1 so it can reach DEV2 dev2_side_ip: "192.168.1.11/24" dev1_iface: "br-wan" # DEV2 behind the tunnel dev2_ssh_user: "root" dev2_passfiles: - "basicpass" - "basicpass2" # SSH options used from controller ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" # Image to stage on DEV2 noninvasive we do not run update -w here image_filename: "fox200-2.2.1-r6801.bin" image_md5: "56b7211709de617e058b98d4204e2562" # Optional SHA256; leave empty to skip SHA256 checks image_sha256: "" dev2_image_dir: "/tmp" dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). rebootin: "" # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" pre_tasks: # ------------------------------- Hostname sanity DEV1 ------------------------------- - name: Read DEV1 hostname from env busybox safe ansible.builtin.raw: > {{ pathprefix }} (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") register: dev1_host_read changed_when: false - name: Stop if DEV1 connected hostname differs from inventory ansible.builtin.meta: end_host when: (dev1_host_read.stdout | trim | length > 0) and ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) tasks: # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- - name: Add temporary IP on DEV1 idempotent treat File exists as OK ansible.builtin.raw: > {{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} register: add_ip changed_when: add_ip.rc == 0 failed_when: > add_ip.rc != 0 and ('File exists' not in (add_ip.stdout | default(''))) and ('File exists' not in (add_ip.stderr | default(''))) - name: Debug add ip results ansible.builtin.debug: msg: - "add_ip.rc={{ add_ip.rc | default('') }}" - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb ansible.builtin.raw: > {{ pathprefix }} bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 register: dev2_mac_scan changed_when: false - name: Set dev2_mac from fdb scan (if any) ansible.builtin.set_fact: dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" changed_when: false - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) ansible.builtin.raw: > {{ pathprefix }} ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true register: dev2_arp_del changed_when: false failed_when: false - name: Add static ARP for DEV2 on DEV1 ansible.builtin.raw: > {{ pathprefix }} ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent register: dev2_arp_add changed_when: dev2_arp_add.rc == 0 failed_when: > dev2_arp_add.rc != 0 and ('File exists' not in (dev2_arp_add.stdout | default(''))) and ('File exists' not in (dev2_arp_add.stderr | default(''))) - name: Debug static ARP result ansible.builtin.debug: msg: - "dev2_mac={{ dev2_mac | default('UNSET') }}" - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" - name: Skip ARP add because MAC not found when: dev2_mac is not defined or dev2_mac | length == 0 ansible.builtin.debug: msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" # ---------------------------- Local tunnel preparation ---------------------------- - name: Pick a free local port for the tunnel controller delegate_to: localhost ansible.builtin.shell: | set -e pick() { for i in $(seq 1 25); do p="$(shuf -i 20000-39999 -n 1)" if command -v ss >/dev/null 2>&1; then if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; return 0 fi else if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then echo "$p"; return 0 fi fi done return 1 } pick register: pick_port changed_when: false - name: Stop if no free local port ansible.builtin.meta: end_host when: (pick_port.stdout | trim | length) == 0 - name: Record chosen local port and control directory delegate_to: localhost ansible.builtin.set_fact: _local_port: "{{ pick_port.stdout | trim }}" _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" - name: Build tunnel control socket path delegate_to: localhost ansible.builtin.set_fact: _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" - name: Debug chosen tunnel params delegate_to: localhost ansible.builtin.debug: msg: - "local_port={{ _local_port }}" - "ctrl_dir={{ _ctrl_dir }}" - "ctrl_sock={{ _ctrl_sock }}" - name: issue arping with own IP ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 # ---------------------------- Start SSH local forward via DEV1 ---------------------------- - name: Start SSH tunnel via DEV1 background ControlMaster delegate_to: localhost ansible.builtin.shell: | set -e USER="{{ dev1_user }}" HOST="{{ ansible_host | default(inventory_hostname) }}" sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ -M -S "{{ _ctrl_sock }}" \ -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ "${USER}@${HOST}" args: executable: /bin/bash register: start_tunnel changed_when: true - name: Small delay for tunnel to settle delegate_to: localhost ansible.builtin.wait_for: timeout: 1 changed_when: false - name: Verify tunnel master running proper O check with destination delegate_to: localhost ansible.builtin.shell: | set -e HOST="{{ ansible_host | default(inventory_hostname) }}" ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true register: tun_check changed_when: false - name: Debug tunnel check delegate_to: localhost ansible.builtin.debug: msg: - "tunnel_check.rc={{ tun_check.rc }}" - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- - name: Sanity confirm tunnel TCP reachability to DEV2 delegate_to: localhost ansible.builtin.shell: | set -e nc -z -w5 127.0.0.1 "{{ _local_port }}" register: nc_probe changed_when: false ignore_errors: true - name: Debug reachability result delegate_to: localhost ansible.builtin.debug: msg: - "nc.rc={{ nc_probe.rc }}" - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" - name: Stop if tunnel TCP check failed ansible.builtin.meta: end_host when: nc_probe.rc != 0 - name: Sanity show presence and permissions of passfiles on controller delegate_to: localhost ansible.builtin.shell: | set -e ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" register: dev2_ls changed_when: false - name: Debug passfiles listing delegate_to: localhost ansible.builtin.debug: msg: - "{{ (dev2_ls.stdout | default('')) | trim }}" - "{{ (dev2_ls.stderr | default('')) | trim }}" # ---------------------------- Pick DEV2 password root only ---------------------------- - name: Try DEV2 login with basicpass root delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f basicpass ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 register: dev2_try_basicpass changed_when: false ignore_errors: true - name: Select basicpass as working passfile when: dev2_try_basicpass.rc == 0 delegate_to: localhost ansible.builtin.set_fact: dev2_passfile_used: "basicpass" changed_when: false - name: Try DEV2 login with basicpass2 root only if first failed when: dev2_passfile_used is not defined delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f basicpass2 ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 register: dev2_try_basicpass2 changed_when: false ignore_errors: true - name: Select basicpass2 as working passfile when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 delegate_to: localhost ansible.builtin.set_fact: dev2_passfile_used: "basicpass2" changed_when: false - name: Set dev2_passfile_used to NONE if neither worked when: dev2_passfile_used is not defined delegate_to: localhost ansible.builtin.set_fact: dev2_passfile_used: "NONE" changed_when: false - name: Debug selected DEV2 passfile or NONE delegate_to: localhost ansible.builtin.debug: msg: - "dev2_passfile_used={{ dev2_passfile_used }}" - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" - name: Read DEV2 hostname via tunnel (busybox-safe) when: dev2_passfile_used != "NONE" delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" args: executable: /bin/bash register: dev2_host_read changed_when: false - name: Normalize hostnames for strict compare ansible.builtin.set_fact: _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" - name: Debug hostname bytes (hex) delegate_to: localhost ansible.builtin.debug: msg: - "inv={{ _inv_hn | tojson }}" - "dev1={{ _dev1_hn | tojson }}" - "dev2={{ _dev2_hn | tojson }}" # ---------------- Hostname equality guard (soft-journal and stop) ---------------- - name: Enforce DEV2 hostname equals inventory and DEV1 block: - name: Fail if DEV2 hostname differs from inventory/DEV1 ansible.builtin.fail: msg: > Hostname mismatch: DEV2='{{ _dev2_hn }}', inventory='{{ _inv_hn }}', DEV1='{{ _dev1_hn }}' when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) rescue: - name: Initialize journal array for hostname mismatch ansible.builtin.set_fact: _journal: [] _blocked: true _prep_blocked: false delegate_to: localhost - name: Append hostname mismatch to journal ansible.builtin.set_fact: _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" delegate_to: localhost - name: Build control queue payload for indoor aborted journal (hostname) ansible.builtin.set_fact: journal_indoor_aborted: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} delegate_to: localhost - name: Publish indoor aborted journal to control queue (hostname) ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_indoor_aborted | to_json }}" payload_encoding: "string" register: rmq_journal_indoor_aborted_hn_resp changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_indoor_aborted_hn_resp.status != 200) or (rmq_journal_indoor_aborted_hn_resp.json is not defined) or (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) delegate_to: localhost - name: Stop host after hostname mismatch ansible.builtin.meta: end_host # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- - name: Initialize journal and flags ansible.builtin.set_fact: _journal: [] _prep_blocked: false _blocked: false - name: Build specific marker filename (no .bin) ansible.builtin.set_fact: _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" - name: Check for any preparation markers on DEV2 (wildcard count) when: dev2_passfile_used != "NONE" delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "ls /tmp/prepared_for* 2>/dev/null | wc -l" args: executable: /bin/bash register: dev2_prep_count changed_when: false ignore_errors: true - name: Soft-block if any prep markers already present when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) ansible.builtin.set_fact: _prep_blocked: true _blocked: true _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" # ---------------------------- DEV2 version firmux primary check ---------------------------- - name: Read DEV2 firmux release when: dev2_passfile_used != "NONE" delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "cat /usr/lib/release/firmux 2>/dev/null || true" args: executable: /bin/bash register: dev2_firmux changed_when: false ignore_errors: true - name: Debug firmux content DEV2 when: dev2_firmux is defined delegate_to: localhost ansible.builtin.debug: msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- - name: "Normalize rebootin (phase 1: raw/is_now/is_int/HOURS)" delegate_to: localhost ansible.builtin.set_fact: _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" _reboot_hours: >- {{ 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) }} - name: "Normalize rebootin (phase 2: requested flag)" delegate_to: localhost ansible.builtin.set_fact: _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" # Compute effective seconds from HOURS (0h → ~20 seconds grace); clamp non-negative - name: "Normalize rebootin (phase 3: compute seconds from HOURS)" delegate_to: localhost ansible.builtin.set_fact: _reboot_seconds: >- {{ 20 if (_reboot_requested | default(false)) and ((_reboot_hours | int) <= 0) else (((_reboot_hours | int) * 3600)) if (_reboot_requested | default(false)) else 0 }} # Derive minutes from seconds (non-negative), for tasks/summary that expect minutes - name: "Normalize rebootin (phase 4: derive minutes)" delegate_to: localhost ansible.builtin.set_fact: _reboot_minutes: "{{ ((_reboot_seconds | int) // 60) if (_reboot_seconds | int) > 0 else 0 }}" # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- - name: Build control queue payload for indoor start journal ansible.builtin.set_fact: journal_indoor_start: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- Indoor: Dev2 is reachable, starting with update. image={{ image_filename }}, md5={{ image_md5 }}, firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} when: dev2_passfile_used != "NONE" delegate_to: localhost - name: Publish indoor start journal to control queue ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_indoor_start | to_json }}" payload_encoding: "string" register: rmq_journal_indoor_start_resp changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_indoor_start_resp.status != 200) or (rmq_journal_indoor_start_resp.json is not defined) or (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) when: journal_indoor_start is defined delegate_to: localhost # ---------------------------- Stage image on DEV2 no write ---------------------------- - name: Check local presence of image file delegate_to: localhost ansible.builtin.stat: path: "{{ image_filename }}" register: local_img - name: Soft-block if image missing locally when: not local_img.stat.exists ansible.builtin.set_fact: _blocked: true _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" - name: Compute local md5 of the image when: local_img.stat.exists delegate_to: localhost ansible.builtin.shell: | set -e md5sum "{{ image_filename }}" | awk '{print $1}' register: local_md5 changed_when: false - name: Verify local md5 matches expected when: local_img.stat.exists delegate_to: localhost ansible.builtin.assert: that: - (local_md5.stdout | trim) == image_md5 fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" success_msg: "Local md5 matches expected" # Optional SHA256 local - name: Compute local sha256 of the image (if provided) when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 delegate_to: localhost ansible.builtin.shell: | set -e sha256sum "{{ image_filename }}" | awk '{print $1}' register: local_sha256 changed_when: false ignore_errors: true - name: Soft-block if local sha256 mismatch or unavailable when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) ansible.builtin.set_fact: _prep_blocked: true _blocked: true _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" # fw_printenv health before upload (soft-fail) - name: Read fw_printenv size (line count) on DEV2 when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "(fw_printenv 2>/dev/null | wc -l) || echo 0" args: executable: /bin/bash register: dev2_fwenv_wc changed_when: false ignore_errors: true - name: Soft-block if fw_printenv too small when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) ansible.builtin.set_fact: _prep_blocked: true _blocked: true _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" - name: Check existing image md5 on DEV2 NOFILE if absent when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" args: executable: /bin/bash register: dev2_md5_before changed_when: false - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" scp \ -P "$PORT" \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" args: executable: /bin/bash register: scp_push changed_when: true - name: Compute md5 of image on DEV2 after copy or if exists when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" args: executable: /bin/bash register: dev2_md5_after changed_when: false - name: Verify DEV2 md5 matches expected (soft-fail journal) when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) ansible.builtin.set_fact: _prep_blocked: true _blocked: true _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" # Optional SHA256 remote - name: Compute sha256 of image on DEV2 (if provided) when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" args: executable: /bin/bash register: dev2_sha256_after changed_when: false ignore_errors: true - name: Soft-block if DEV2 sha256 mismatch/unavailable when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) ansible.builtin.set_fact: _prep_blocked: true _blocked: true _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" - name: Run noninvasive update check on DEV2 update c when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=15 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "update -c '{{ dev2_image_path }}' 2>&1 || true" args: executable: /bin/bash register: dev2_update_check changed_when: false - name: Debug update c output DEV2 when: dev2_update_check is defined delegate_to: localhost ansible.builtin.debug: msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" - name: Soft-block if update -c did not say valid when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) ansible.builtin.set_fact: _prep_blocked: true _blocked: true _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- - name: Build control queue payload for indoor aborted journal ansible.builtin.set_fact: journal_indoor_aborted: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} when: (_blocked | default(false)) delegate_to: localhost - name: Publish indoor aborted journal to control queue ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_indoor_aborted | to_json }}" payload_encoding: "string" register: rmq_journal_indoor_aborted_resp changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_indoor_aborted_resp.status != 200) or (rmq_journal_indoor_aborted_resp.json is not defined) or (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) when: journal_indoor_aborted is defined delegate_to: localhost # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ - name: Upgrade write and flip block when: not (_blocked | default(false)) block: - name: Write image on DEV2 (this will take a while) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=0 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "update -w '{{ dev2_image_path }}'" args: executable: /bin/bash register: dev2_up_write changed_when: true failed_when: dev2_up_write.stdout is not search('update is complete') - name: Read current active partition on DEV2 delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "fw_printenv active | awk -F= '/^active=/{print \$2}'" args: executable: /bin/bash register: dev2_active_before changed_when: false failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] - name: Determine new active value for DEV2 when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) ansible.builtin.set_fact: dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" - name: Switch active partition to {{ dev2_new_active }} when: dev2_new_active is defined delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "fw_setenv active {{ dev2_new_active }}" args: executable: /bin/bash register: dev2_setenv_out changed_when: true - name: Verify active partition flipped on DEV2 when: dev2_new_active is defined delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "fw_printenv active | awk -F= '/^active=/{print \$2}'" args: executable: /bin/bash register: dev2_active_after changed_when: false failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) - name: Create specific prep marker on DEV2 (no generic) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ "touch '{{ _marker_specific }}'" args: executable: /bin/bash register: dev2_marker_write changed_when: true ignore_errors: true # ---------------------------- Reboot scheduling (normalized) ---------------------------- - name: Schedule delayed reboot on DEV2 minutes when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) delegate_to: localhost ansible.builtin.shell: | set -e PORT="{{ _local_port }}" SECS="{{ _reboot_seconds | int }}" CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' sshpass -f "{{ dev2_passfile_used }}" ssh \ -o AddressFamily=inet \ -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ -o ConnectTimeout=10 \ -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" args: executable: /bin/bash register: dev2_reboot_sched changed_when: true ignore_errors: true - name: Build control queue payload for indoor-restart-scheduled tag ansible.builtin.set_fact: tag_restart_sched_payload: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "tag_add" task_result: "indoor-restart-scheduled" when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) delegate_to: localhost - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ tag_restart_sched_payload | to_json }}" payload_encoding: "string" register: rmq_tag_restart_sched_resp changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) failed_when: > (rmq_tag_restart_sched_resp.status != 200) or (rmq_tag_restart_sched_resp.json is not defined) or (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) when: tag_restart_sched_payload is defined delegate_to: localhost - name: Note reboot request bad format when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested ansible.builtin.debug: msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- - name: Build control queue payload for indoor updated journal ansible.builtin.set_fact: _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" when: not (_blocked | default(false)) delegate_to: localhost - name: Build indoor updated journal payload text ansible.builtin.set_fact: journal_indoor_updated: inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" task_name: "journal_add" task_result: >- Indoor: Dev2 updated, scheduling reboot {{ ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) else 'not requested' }}. write_done={{ _write_success }}, active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} when: not (_blocked | default(false)) and (_write_success | bool) delegate_to: localhost - name: Publish indoor updated journal to control queue ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" routing_key: "{{ control_queue }}" payload: "{{ journal_indoor_updated | to_json }}" payload_encoding: "string" register: rmq_journal_indoor_updated_resp changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) failed_when: > (rmq_journal_indoor_updated_resp.status != 200) or (rmq_journal_indoor_updated_resp.json is not defined) or (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) when: journal_indoor_updated is defined delegate_to: localhost # ──────────────────────────────── NEW: derive expected version for the checker ──────────────────────────────── # >>> ADDED: initialize scheduling window so au_delay_sec exists for the derivation <<< - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) ansible.builtin.set_fact: au_attempt: 1 au_max_attempts: 3 au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" when: not (_blocked | default(false)) and (_write_success | bool) delegate_to: localhost - name: Generate correlation ID and original emitted timestamp ansible.builtin.set_fact: au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" when: au_delay_sec is defined delegate_to: localhost # <<< END ADDED >>> # --- SAFE expected target derivation (split into separate set_fact tasks) --- - name: Derive expected indoor firmware from image filename (safe – base) delegate_to: localhost ansible.builtin.set_fact: _img_base: "{{ image_filename | regex_replace('\\.bin$','') }}" when: au_delay_sec is defined - name: Derive X.Y.Z core from image filename (if any) delegate_to: localhost ansible.builtin.set_fact: _img_ver_core: >- {{ (_img_base is search('[0-9]+\\.[0-9]+\\.[0-9]+')) | ternary( (_img_base | regex_replace('.*?([0-9]+\\.[0-9]+\\.[0-9]+).*','\\1')), '' ) }} when: au_delay_sec is defined - name: Derive rev number from -rNNNN (preferred) delegate_to: localhost ansible.builtin.set_fact: _img_rev_num: >- {{ (_img_base is search('-r[0-9]+')) | ternary( (_img_base | regex_replace('.*-r([0-9]+).*','\\1')), '' ) }} when: au_delay_sec is defined - name: If no -rNNNN, try 'rev NNNN' (case-insensitive) delegate_to: localhost ansible.builtin.set_fact: _img_rev_num: >- {{ (_img_rev_num | length) > 0 | ternary( _img_rev_num, ( (_img_base is search('(?i)rev[^0-9]*[0-9]+')) | ternary( (_img_base | regex_replace('.*(?i)rev[^0-9]*([0-9]+).*','\\1')), '' ) ) ) }} when: au_delay_sec is defined - name: Assemble expected_fw_core (prefer X.Y.Z-rNNNN → X.Y.Z → image base) delegate_to: localhost ansible.builtin.set_fact: expected_fw_core: >- {{ (_img_ver_core | length) > 0 and (_img_rev_num | length) > 0 and (_img_ver_core ~ '-r' ~ _img_rev_num) or ((_img_ver_core | length) > 0 and _img_ver_core) or _img_base }} when: au_delay_sec is defined - name: Debug derived target version delegate_to: localhost ansible.builtin.debug: msg: - "image_filename={{ image_filename }}" - "expected_fw_core={{ expected_fw_core }}" when: expected_fw_core is defined - name: If no -rNNNN, also try to pull rev from 'rev NNNN' (case-insensitive) delegate_to: localhost ansible.builtin.set_fact: _img_rev_num: >- {{ (_img_rev_num | length) > 0 | ternary( _img_rev_num, ( (_img_base is search('(?i)rev[^0-9]*[0-9]+')) | ternary( (_img_base | regex_replace('.*(?i)rev[^0-9]*([0-9]+).*','\\1')), '' ) ) ) }} when: au_delay_sec is defined - name: Assemble expected_fw_core (prefer X.Y.Z-rNNNN → X.Y.Z → image base) delegate_to: localhost ansible.builtin.set_fact: expected_fw_core: >- {{ (_img_ver_core | length) > 0 and (_img_rev_num | length) > 0 and (_img_ver_core ~ '-r' ~ _img_rev_num) or ((_img_ver_core | length) > 0 and _img_ver_core) or _img_base }} when: au_delay_sec is defined - name: Debug derived target version delegate_to: localhost ansible.builtin.debug: msg: - "image_filename={{ image_filename }}" - "expected_fw_core={{ expected_fw_core }}" when: expected_fw_core is defined - name: Build after-upgrade check payload (attempt 1) ansible.builtin.set_fact: afterupgrade_payload: task_name: "afterupgrade_indoor_check" inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" target_version: "{{ expected_fw_core | default(fw_banner_repr | default(image_filename | regex_replace('\\.bin$',''))) }}" attempt: "{{ au_attempt }}" max_attempts: "{{ au_max_attempts }}" current_delay_sec: "{{ au_delay_sec }}" correlation_id: "{{ au_correlation_id }}" original_emitted_at: "{{ au_original_emitted_at }}" schema_version: 1 when: au_delay_sec is defined delegate_to: localhost - name: Debug x-delay about to be sent (ms) ansible.builtin.debug: msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" when: afterupgrade_payload is defined delegate_to: localhost - name: Publish delayed after-upgrade check to holding exchange ansible.builtin.uri: url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" method: POST user: "{{ rmq_user }}" password: "{{ rmq_pass }}" force_basic_auth: true status_code: 200 headers: content-type: "application/json" body_format: json body: properties: content_type: "application/json" headers: x-delay: "{{ (au_delay_sec | int) * 1000 }}" routing_key: "{{ afterupgrade_routing_key }}" payload: "{{ afterupgrade_payload | to_json }}" payload_encoding: "string" register: rmq_afterupgrade_resp changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) failed_when: > (rmq_afterupgrade_resp.status != 200) or (rmq_afterupgrade_resp.json is not defined) when: afterupgrade_payload is defined delegate_to: localhost # ---------------------------- summary ---------------------------- - name: Summary show key results delegate_to: localhost ansible.builtin.debug: msg: - "dev2_passfile_used={{ dev2_passfile_used }}" - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" - "local_image_present={{ local_img.stat.exists | default(false) }}" - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" - "reboot_requested={{ _reboot_requested | default(false) }}" - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" - "prep_blocked={{ _prep_blocked | default(false) }}" - "blocked={{ _blocked | default(false) }}" - "journal={{ (_journal | default([])) | join(' || ') }}" post_tasks: - name: Cleanup always block: - ansible.builtin.debug: msg: "entering cleanup block" changed_when: false delegate_to: localhost always: - name: Close tunnel best effort delegate_to: localhost ansible.builtin.shell: | ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true changed_when: false ignore_errors: true - name: Remove control dir best effort delegate_to: localhost ansible.builtin.file: path: "{{ _ctrl_dir | default('/tmp/none') }}" state: absent ignore_errors: true - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK ansible.builtin.raw: > {{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} register: del_ip changed_when: del_ip.rc == 0 failed_when: > del_ip.rc != 0 and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) - name: Debug del ip results ansible.builtin.debug: msg: - "del_ip.rc={{ del_ip.rc | default('') }}" - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" when: del_ip is defined