commit 110301f86255cd892d55d53c649271d683912c86 Author: ansible user Date: Wed Oct 22 14:11:49 2025 +0200 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c2efa39 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +### FROM ubuntu:24.04 +FROM public.ecr.aws/docker/library/ubuntu:24.04 + + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Africa/Johannesburg + +# OS deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + tzdata \ + python3 python3-venv python3-pip \ + openssh-client sshpass netcat-traditional \ + vim gawk \ + jq curl ca-certificates git \ + && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone \ + && dpkg-reconfigure -f noninteractive tzdata \ + && rm -rf /var/lib/apt/lists/* + +ENV APP_DIR=/opt/containers/ansible-worker +ENV DATA_DIR=${APP_DIR}/data +ENV VENV_DIR=${DATA_DIR}/ansible_venv + +# Copy your Ansible project files into the image (immutable) +# Expecting host: ./files/{ansible.cfg, inventory/, group_vars/, ...} +COPY files/ ${APP_DIR}/app/ + +# Entry script will bootstrap the venv into DATA_DIR if missing +COPY files/entrypoint.sh /usr/local/bin/ansible-entrypoint +RUN chmod +x /usr/local/bin/ansible-entrypoint + +# Default config paths +ENV ANSIBLE_CONFIG=${APP_DIR}/app/ansible.cfg +# Optional: speed up password auth on Dropbear/OpenWrt devices +ENV ANSIBLE_SSH_ARGS="-o PubkeyAuthentication=no" + +# put the venv on PATH for all processes (incl. docker exec sessions) +ENV PATH="/opt/containers/ansible-worker/data/ansible_venv/bin:${PATH}" + +# keep collections in persistent storage +#ENV ANSIBLE_COLLECTIONS_PATHS="/opt/containers/ansible-worker/data/collections:/usr/share/ansible/collections" +ENV ANSIBLE_COLLECTIONS_PATH="/opt/containers/ansible-worker/data/collections:/usr/share/ansible/collections" + +ENV QUEUE="queue_deviceconfig,queue_persuasive" + +# Work from the "app" dir; data/ is mounted separately +WORKDIR ${APP_DIR}/app + +# Keep container alive by default; entrypoint sets up venv+PATH +ENTRYPOINT ["/usr/local/bin/ansible-entrypoint"] +#CMD ["sleep", "infinity"] + +# …or, if it’s under bin and named rabbitmq-client.sh as you hinted: +CMD ["/opt/containers/ansible-worker/app/bin/rabbitmq-client.sh"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e1325ba --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +services: + ansible-worker: + build: + context: . + dockerfile: Dockerfile + image: ansible-worker:latest + # container_name: ansible-worker # <<< REMOVE THIS + + working_dir: /opt/containers/ansible-worker/app + entrypoint: ["/opt/containers/ansible-worker/files/entrypoint.sh"] + command: ["/opt/containers/ansible-worker/app/bin/rabbit-client.sh"] + + restart: unless-stopped + tty: true + stdin_open: true + + environment: + ANSIBLE_CONFIG: /opt/containers/ansible-worker/app/ansible.cfg + ANSIBLE_SSH_ARGS: "-o PubkeyAuthentication=no" + TZ: Africa/Johannesburg + + volumes: + - ./data:/opt/containers/ansible-worker/data:rw + - ./files:/opt/containers/ansible-worker/files:rw diff --git a/docker-compose.yml-backup b/docker-compose.yml-backup new file mode 100644 index 0000000..452647b --- /dev/null +++ b/docker-compose.yml-backup @@ -0,0 +1,21 @@ +services: + ansible-worker: + build: + context: . + dockerfile: Dockerfile + image: ansible-worker:latest + container_name: ansible-worker + working_dir: /opt/containers/ansible-worker/app + entrypoint: ["/usr/local/bin/ansible-entrypoint"] + command: ["sleep", "infinity"] + restart: unless-stopped + tty: true + stdin_open: true + environment: + ANSIBLE_CONFIG: /opt/containers/ansible-worker/app/ansible.cfg + ANSIBLE_SSH_ARGS: "-o PubkeyAuthentication=no" + # bind-mount only DATA; files/ are baked into the image + volumes: + - ./data:/opt/containers/ansible-worker/data:rw + - ./files:/opt/containers/ansible-worker/files:rw + diff --git a/docker-compose.yml-standalone-runner b/docker-compose.yml-standalone-runner new file mode 100644 index 0000000..212f195 --- /dev/null +++ b/docker-compose.yml-standalone-runner @@ -0,0 +1,29 @@ +services: + ansible-worker: + build: + context: . + dockerfile: Dockerfile + image: ansible-worker:latest + container_name: ansible-worker + + # Keep working dir inside /app + working_dir: /opt/containers/ansible-worker/app + + # 🔑 Use the entrypoint from the bind-mounted ./files + entrypoint: ["/opt/containers/ansible-worker/files/entrypoint.sh"] + # command: ["sleep", "infinity"] + command: ["/opt/containers/ansible-worker/app/bin/rabbit-client.sh"] + + restart: unless-stopped + tty: true + stdin_open: true + + environment: + ANSIBLE_CONFIG: /opt/containers/ansible-worker/app/ansible.cfg + ANSIBLE_SSH_ARGS: "-o PubkeyAuthentication=no" + + volumes: + # Persist venv/collections/etc. + - ./data:/opt/containers/ansible-worker/data:rw + # Expose your files (entrypoint, nbplay, playbooks, etc.) + - ./files:/opt/containers/ansible-worker/files:rw diff --git a/files/2.2.0-r9739.bin b/files/2.2.0-r9739.bin new file mode 100644 index 0000000..4fb6c94 Binary files /dev/null and b/files/2.2.0-r9739.bin differ diff --git a/files/2.2.1-r9763.bin b/files/2.2.1-r9763.bin new file mode 100644 index 0000000..53a1af8 Binary files /dev/null and b/files/2.2.1-r9763.bin differ diff --git a/files/2.2.2-r9778.bin b/files/2.2.2-r9778.bin new file mode 100644 index 0000000..fb34ec7 Binary files /dev/null and b/files/2.2.2-r9778.bin differ diff --git a/files/ansible-playbooks/afterupgrade_check.yml b/files/ansible-playbooks/afterupgrade_check.yml new file mode 100644 index 0000000..9be8afb --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_check.yml @@ -0,0 +1,355 @@ +--- +- name: After-upgrade verification (banner check + reporting) + hosts: all + gather_facts: no + + # RabbitMQ + defaults (match the big script) + vars: + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + rmq_exchange: "controls" + control_queue: "queue_controls" + + # Probing/SSH defaults + tcp_port: 22 + nc_timeout: 5 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + ssh_timeout: 10 + + # Do NOT self-reference max_attempts. We’ll normalize below. + max_attempts_default: 3 + + tasks: + # ---- Normalize metadata safely (no self-referential defaults) ---- + - name: Normalize metadata (no clever transforms) + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + # preserve the original string verbatim for all subsequent retries + target_version_full: "{{ target_version | default('') }}" + + - name: Show received metadata + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "max_attempts={{ effective_max_attempts }}" + - "correlation_id={{ correlation_id }}" + - "original_emitted_at={{ original_emitted_at }}" + - "target_version(full)={{ target_version_full }}" + + # ---- Fast TCP reachability probe (controller-side) ---- + - name: Check if TCP/{{ tcp_port }} is reachable with nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Build failure journal (no TCP connectivity) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed). + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (no TCP connectivity) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). + - name: Stop host after TCP failure (we’ll schedule or close out below) + when: nc_probe.rc != 0 + ansible.builtin.meta: noop + + # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- + - name: Probe banner via SSH from controller (classic extraction) + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ ssh_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ ssh_pass }}' \ + ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ + "${USER}@${HOST}" \ + "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" + register: banner_probe + changed_when: false + ignore_errors: true + + - name: Show the current version (banner line) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.debug: + msg: "{{ banner_probe.stdout | trim }}" + + - name: Evaluate version match (full-string contains check) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.set_fact: + version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" + + # ---- Journaling paths ---- + # Success: banner matches expected full target_version + - name: Build success journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_success + changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) + + # NEW: send a control tag to clean up device state on success + - name: Build cleanup control payload (update_cleanup_success) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + control_cleanup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "update_cleanup_success" + + - name: Publish cleanup control message to control queue + when: control_cleanup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ control_cleanup_payload | to_json }}" + payload_encoding: "string" + register: rmq_cleanup_success + changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) + + # Mismatch: reachable & banner read, but not equal to target_version + - name: Build mismatch journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_mismatch_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish mismatch journal to control queue + when: journal_mismatch_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_mismatch_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_mismatch + changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) + + # SSH error path: TCP OK, but SSH failed + - name: Build failure journal payload (ssh error) + mark retry + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): + {{ (banner_probe.stderr | default('') | trim) }} + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (ssh error) to control queue + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_fail + changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) + + # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- + - name: Compute next-attempt delay (ms) according to policy + when: (_needs_retry | default(false)) | bool + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: >- + {% if attempt | int == 1 %} + 300 + {% elif attempt | int == 2 %} + 600 + {% else %} + 0 + {% endif %} + next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" + + # If we've reached the cap, send a final “gave up” journal and stop. + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + # Otherwise schedule the next attempt (only if we still have budget) + - name: Build delayed after-upgrade payload for next attempt + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host diff --git a/files/ansible-playbooks/afterupgrade_check.yml-from_chat b/files/ansible-playbooks/afterupgrade_check.yml-from_chat new file mode 100644 index 0000000..d7edd1b --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_check.yml-from_chat @@ -0,0 +1,355 @@ +--- +- name: After-upgrade verification (banner check + reporting) + hosts: all + gather_facts: no + + # RabbitMQ + defaults (match the big script) + vars: + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + rmq_exchange: "controls" + control_queue: "queue_controls" + + # Probing/SSH defaults + tcp_port: 22 + nc_timeout: 5 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + ssh_timeout: 10 + + # Do NOT self-reference max_attempts. We’ll normalize below. + max_attempts_default: 3 + + tasks: + # ---- Normalize metadata safely (no self-referential defaults) ---- + - name: Normalize metadata (no clever transforms) + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + # preserve the original string verbatim for all subsequent retries + target_version_full: "{{ target_version | default('') }}" + + - name: Show received metadata + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "max_attempts={{ effective_max_attempts }}" + - "correlation_id={{ correlation_id }}" + - "original_emitted_at={{ original_emitted_at }}" + - "target_version(full)={{ target_version_full }}" + + # ---- Fast TCP reachability probe (controller-side) ---- + - name: Check if TCP/{{ tcp_port }} is reachable with nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Build failure journal (no TCP connectivity) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed). + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (no TCP connectivity) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). + - name: Stop host after TCP failure (we’ll schedule or close out below) + when: nc_probe.rc != 0 + ansible.builtin.meta: noop + + # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- + - name: Probe banner via SSH from controller (classic extraction) + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ ssh_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ ssh_pass }}' \ + ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ + "${USER}@${HOST}" \ + "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" + register: banner_probe + changed_when: false + ignore_errors: true + + - name: Show the current version (banner line) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.debug: + msg: "{{ banner_probe.stdout | trim }}" + + - name: Evaluate version match (full-string contains check) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.set_fact: + version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" + + # ---- Journaling paths ---- + # Success: banner matches expected full target_version + - name: Build success journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_full }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_success + changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) + + # NEW: send a control tag to clean up device state on success + - name: Build cleanup control payload (update_cleanup_success) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + control_cleanup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "update_cleanup_success" + + - name: Publish cleanup control message to control queue + when: control_cleanup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ control_cleanup_payload | to_json }}" + payload_encoding: "string" + register: rmq_cleanup_success + changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) + + # Mismatch: reachable & banner read, but not equal to target_version + - name: Build mismatch journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_mismatch_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish mismatch journal to control queue + when: journal_mismatch_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_mismatch_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_mismatch + changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) + + # SSH error path: TCP OK, but SSH failed + - name: Build failure journal payload (ssh error) + mark retry + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): + {{ (banner_probe.stderr | default('') | trim) }} + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (ssh error) to control queue + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_fail + changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) + + # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- + - name: Compute next-attempt delay (ms) according to policy + when: (_needs_retry | default(false)) | bool + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: >- + {% if attempt | int == 1 %} + 300 + {% elif attempt | int == 2 %} + 600 + {% else %} + 0 + {% endif %} + next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" + + # If we've reached the cap, send a final “gave up” journal and stop. + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + # Otherwise schedule the next attempt (only if we still have budget) + - name: Build delayed after-upgrade payload for next attempt + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host diff --git a/files/ansible-playbooks/afterupgrade_check.yml-from_container b/files/ansible-playbooks/afterupgrade_check.yml-from_container new file mode 100644 index 0000000..03c5f29 --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_check.yml-from_container @@ -0,0 +1,432 @@ +--- +- name: After-upgrade verification (banner check + reporting) + hosts: all + gather_facts: no + + # RabbitMQ + defaults (match the big script) + vars: + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + rmq_exchange: "controls" + control_queue: "queue_controls" + + # Probing/SSH defaults + tcp_port: 22 + nc_timeout: 5 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + ssh_timeout: 10 + + # Do NOT self-reference max_attempts. We’ll normalize below. + max_attempts_default: 3 + + tasks: + # ---- Normalize metadata safely (no self-referential defaults) ---- + - name: Normalize metadata (no clever transforms) + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + # preserve the original string verbatim for all subsequent retries + target_version_full: "{{ target_version | default('') }}" + + - name: Show received metadata + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "max_attempts={{ effective_max_attempts }}" + - "correlation_id={{ correlation_id }}" + - "original_emitted_at={{ original_emitted_at }}" + - "target_version(full)={{ target_version_full }}" + + # ---- Fast TCP reachability probe (controller-side) ---- + - name: Check if TCP/{{ tcp_port }} is reachable with nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Build failure journal (no TCP connectivity) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): + TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }} + Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (no TCP connectivity) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). + - name: Stop host after TCP failure (we’ll schedule or close out below) + when: nc_probe.rc != 0 + ansible.builtin.meta: noop + + # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- + - name: Probe banner via SSH from controller (classic extraction) + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ ssh_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ ssh_pass }}' \ + ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ + "${USER}@${HOST}" \ + "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" + register: banner_probe + changed_when: false + ignore_errors: true + + - name: Show the current version (banner line) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.debug: + msg: "{{ banner_probe.stdout | trim }}" + + - name: Evaluate version match (full-string contains check) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.set_fact: + version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" + + # ---- Journaling paths ---- + + # Success: banner matches expected full target_version + - name: Build success journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + Banner='{{ (banner_probe.stdout | default('') | trim) }}' + Target='{{ target_version_full }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_success + changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) + + # NEW: send a control tag to clean up device state on success + - name: Build cleanup control payload (update_cleanup_success) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + control_cleanup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "update_cleanup_success" + + - name: Publish cleanup control message to control queue + when: control_cleanup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ control_cleanup_payload | to_json }}" + payload_encoding: "string" + register: rmq_cleanup_success + changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) + + # --- NEW: on success, set NetBox custom field "indoor_fwver" and remove the "indoor-restart-scheduled" tag --- + + - name: Normalize DEV2 firmware string for custom field (e.g., "2.2.1 rev 6801" -> "2.2.1-r6801") + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.set_fact: + indoor_fwver_norm: >- + {{ + ( (dev2_fwver.stdout | default('') | trim) + | regex_replace('\\s*[Rr][Ee][Vv]\\s*(\\d+)', '-r\\1') + | regex_replace('\\s+', ' ') + | trim + ) + }} + + - name: Publish NetBox custom field indoor_fwver to control queue + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'indoor_fwver', + 'task_result': (indoor_fwver_norm | default('')) + } | to_json }}" + payload_encoding: "string" + register: rmq_cf_indoor_fwver + changed_when: (rmq_cf_indoor_fwver.json is defined) and (rmq_cf_indoor_fwver.json.routed | default(false) | bool) + + - name: Build payload to remove indoor-restart-scheduled tag (DEV1) + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.set_fact: + tag_remove_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_remove" + task_result: "indoor-restart-scheduled" + + - name: Publish indoor-restart-scheduled tag removal + when: tag_remove_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_remove_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_remove_sched + changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) + + + # Mismatch: reachable & banner read, but not equal to target_version + - name: Build mismatch journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_mismatch_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish mismatch journal to control queue + when: journal_mismatch_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_mismatch_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_mismatch + changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) + + # SSH error path: TCP OK, but SSH failed + - name: Build failure journal payload (ssh error) + mark retry + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): + {{ (banner_probe.stderr | default('') | trim) }} + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (ssh error) to control queue + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_fail + changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) + + # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- + + - name: Compute next-attempt delay (ms) according to policy + when: (_needs_retry | default(false)) | bool + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: >- + {% if attempt | int == 1 %} + 300 + {% elif attempt | int == 2 %} + 600 + {% else %} + 0 + {% endif %} + next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" + + # If we've reached the cap, send a final “gave up” journal and stop. + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + # Otherwise schedule the next attempt (only if we still have budget) + - name: Build delayed after-upgrade payload for next attempt + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host + diff --git a/files/ansible-playbooks/afterupgrade_check.yml-screewed b/files/ansible-playbooks/afterupgrade_check.yml-screewed new file mode 100644 index 0000000..e5cd2ce --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_check.yml-screewed @@ -0,0 +1,359 @@ +--- +- name: After-upgrade verification (banner check + reporting) + hosts: all + gather_facts: no + + # RabbitMQ + defaults (match the big script) + vars: + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + rmq_exchange: "controls" + control_queue: "queue_controls" + + # Probing/SSH defaults + tcp_port: 22 + nc_timeout: 5 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + ssh_timeout: 10 + + # Do NOT self-reference max_attempts. We’ll normalize below. + max_attempts_default: 3 + + tasks: + # ---- Normalize metadata safely (no self-referential defaults) ---- + - name: Normalize metadata (no clever transforms) + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + # preserve the original string verbatim for all subsequent retries + target_version_full: "{{ target_version | default('') }}" + + - name: Show received metadata + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "max_attempts={{ effective_max_attempts }}" + - "correlation_id={{ correlation_id }}" + - "original_emitted_at={{ original_emitted_at }}" + - "target_version(full)={{ target_version_full }}" + + # ---- Fast TCP reachability probe (controller-side) ---- + - name: Check if TCP/{{ tcp_port }} is reachable with nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Build failure journal (no TCP connectivity) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): + TCP {{ tcp_port }} unreachable (nc failed). Correlation={{ correlation_id }} + Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (no TCP connectivity) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”). + - name: Stop host after TCP failure (we’ll schedule or close out below) + when: nc_probe.rc != 0 + ansible.builtin.meta: noop + + # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ---- + - name: Probe banner via SSH from controller (classic extraction) + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ ssh_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ ssh_pass }}' \ + ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ + "${USER}@${HOST}" \ + "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" + register: banner_probe + changed_when: false + ignore_errors: true + + - name: Show the current version (banner line) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.debug: + msg: "{{ banner_probe.stdout | trim }}" + + - name: Evaluate version match (full-string contains check) + when: nc_probe.rc == 0 and banner_probe.rc == 0 + ansible.builtin.set_fact: + version_match: "{{ (target_version_full | length > 0) and (target_version_full in (banner_probe.stdout | default(''))) }}" + + # ---- Journaling paths ---- + + # Success: banner matches expected full target_version + - name: Build success journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + Banner='{{ (banner_probe.stdout | default('') | trim) }}' + Target='{{ target_version_full }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_success + changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool) + + # NEW: send a control tag to clean up device state on success + - name: Build cleanup control payload (update_cleanup_success) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + control_cleanup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "update_cleanup_success" + + - name: Publish cleanup control message to control queue + when: control_cleanup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ control_cleanup_payload | to_json }}" + payload_encoding: "string" + register: rmq_cleanup_success + changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool) + + # Mismatch: reachable & banner read, but not equal to target_version + - name: Build mismatch journal payload + when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_mismatch_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + Expected='{{ target_version_full }}' Got='{{ (banner_probe.stdout | default('') | trim) }}' + Correlation={{ correlation_id }} Original={{ original_emitted_at }} + + - name: Publish mismatch journal to control queue + when: journal_mismatch_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_mismatch_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_mismatch + changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) + + # SSH error path: TCP OK, but SSH failed + - name: Build failure journal payload (ssh error) + mark retry + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}): + {{ (banner_probe.stderr | default('') | trim) }} + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (ssh error) to control queue + when: nc_probe.rc == 0 and banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_fail + changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool) + + # ---- Retry scheduling (ONLY when we flagged _needs_retry) ---- + + - name: Compute next-attempt delay (ms) according to policy + when: (_needs_retry | default(false)) | bool + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: >- + {% if attempt | int == 1 %} + 300 + {% elif attempt | int == 2 %} + 600 + {% else %} + 0 + {% endif %} + next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}" + + # If we've reached the cap, send a final “gave up” journal and stop. + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + # Otherwise schedule the next attempt (only if we still have budget) + - name: Build delayed after-upgrade payload for next attempt + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after TCP/SSH failure (scheduled next or gave-up already) + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml b/files/ansible-playbooks/afterupgrade_indoor_check.yml new file mode 100644 index 0000000..08ab118 --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_indoor_check.yml @@ -0,0 +1,744 @@ +--- +- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel) + hosts: all + gather_facts: no + + vars: + # ---------------- BusyBox-safe path prefix ---------------- + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # ---------------- DEV1 (outer device) ---------------- + dev1_user: "root" + dev1_pass: "wavewave" + dev1_iface: "br-wan" + + # ---------------- DEV2 (indoor behind DEV1) ---------------- + dev2_host: "192.168.1.1" + dev2_port: 22 + dev2_side_ip: "192.168.1.11/24" + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # ---------------- RabbitMQ (same env scheme as main playbooks) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + tasks: + # ---- Normalize metadata (from delayed message) ---- + - name: Normalize after-upgrade metadata + delegate_to: localhost + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + target_version_full: "{{ target_version | default('') }}" + + # (NEW) Show what we received from the scheduler (for easy troubleshooting) + - name: Debug received scheduler metadata + delegate_to: localhost + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "max_attempts={{ effective_max_attempts }}" + - "correlation_id={{ correlation_id }}" + - "original_emitted_at={{ original_emitted_at }}" + - "target_version(full)={{ target_version_full }}" + + # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ---- + - name: Check if TCP/22 on DEV1 is reachable + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22 + register: nc_probe + changed_when: false + ignore_errors: true + + # ---- If TCP down: journal + schedule next try or give up ---- + - name: Build failure journal (TCP unreachable) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}): + TCP 22 unreachable. Correlation={{ correlation_id }} + Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (TCP unreachable) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + - name: Compute next-attempt delay (10 minutes) and counters + when: (_needs_retry | default(false)) | bool + delegate_to: localhost + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: 600 + next_delay_ms: 600000 + + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path=TCP. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + - name: Build delayed after-upgrade payload for next attempt (10m) + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after scheduling next attempt + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host + + + # ---------------- Step 1: Read DEV1 hostname and sanity ---------------- + - name: Read DEV1 hostname + when: nc_probe.rc == 0 + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 hostname mismatch + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + # ---------------- Step 2: Setup temporary IP for reachability ---------------- + - name: Add temporary IP on DEV1 (ignore if exists) + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 and + ('File exists' not in (add_ip.stdout | default(''))) and + ('File exists' not in (add_ip.stderr | default(''))) + + # ---------------- Step 3: Start SSH tunnel via DEV1 ---------------- + - name: Pick random free local port + delegate_to: localhost + ansible.builtin.shell: | + set -e + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n1)" + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi + done + exit 1 + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control dir + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Start SSH ControlMaster tunnel via DEV1 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sshpass -p '{{ dev1_pass }}' ssh -f -N \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \ + "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) --- + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running (ssh -O check) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + # ---------------- Step 4: Determine working password for DEV2 ---------------- + - name: Try both passfiles for DEV2 + delegate_to: localhost + ansible.builtin.shell: | + for f in {{ dev2_passfiles | join(' ') }}; do + if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then + echo "$f"; exit 0; + fi + done + echo NONE + register: dev2_passfile_try + changed_when: false + + - name: Save selected DEV2 passfile + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}" + changed_when: false + + - name: Debug selected DEV2 passfile + delegate_to: localhost + ansible.builtin.debug: + msg: "dev2_passfile_used={{ dev2_passfile_used }}" + + - name: Stop if no valid passfile found + ansible.builtin.meta: end_host + when: dev2_passfile_used == "NONE" + + # ---------------- Step 5: Read firmware version on DEV2 ---------------- + - name: Read firmware version from DEV2 + delegate_to: localhost + ansible.builtin.shell: | + sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \ + root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown" + register: dev2_fwver + changed_when: false + ignore_errors: true + + - name: Show firmware version readout + delegate_to: localhost + ansible.builtin.debug: + msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" + + # ---------------- Retry metadata + success evaluation (read OK) ---------------- + - name: Normalize retry metadata for indoor checker + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + + - name: Check if firmware read succeeded (read_ok) + delegate_to: localhost + ansible.builtin.set_fact: + read_ok: "{{ (dev2_fwver.rc | default(1) == 0) + and ((dev2_fwver.stdout | default('') | trim) | length > 0) + and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}" + + # ===================== (NEW) Unconditional normalization + comparison ===================== + # Compute expected_norm from target_version_full. If the full filename-like string is sent, + # we try to extract the "X.Y.Z-rNNNN" core; otherwise use the trimmed original. + + - name: Normalize expected target string (step 1: compute components) + delegate_to: localhost + ansible.builtin.set_fact: + expected_norm_step1: "{{ (target_version_full | default('') | trim) }}" + expected_norm_core: >- + {{ + (target_version_full | default('') | + regex_search('([0-9]+\\.[0-9]+\\.[0-9]+-r[0-9]+)', '\\1')) + | default('', true) + }} + + - name: Normalize expected target string (step 2: choose core if present) + delegate_to: localhost + ansible.builtin.set_fact: + expected_norm: "{{ (expected_norm_core | length > 0) | ternary(expected_norm_core, expected_norm_step1) }}" + + # Normalize banner/firmux from DEV2: convert "... rev 6801" → "...-r6801" + - name: Normalize banner/firmux string from DEV2 + delegate_to: localhost + ansible.builtin.set_fact: + banner_raw: "{{ (dev2_fwver.stdout | default('') | trim) }}" + banner_norm: >- + {{ + (banner_raw | lower is search('-r[0-9]+$')) + | ternary(banner_raw, (banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1'))) + }} + + - name: Evaluate version match (normalized equality or contains) + delegate_to: localhost + ansible.builtin.set_fact: + version_match: >- + {{ + (expected_norm | length > 0) + and ( + (banner_norm == expected_norm) + or (banner_norm is search(expected_norm)) + or (expected_norm is search(banner_norm)) + ) + }} + + - name: Debug compare snapshot (expected vs actual normalized) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "expected_norm='{{ expected_norm }}'" + - "banner_norm='{{ banner_norm }}'" + - "version_match={{ version_match | default(false) }}" + + # ===================== Journaling/Tagging paths ===================== + # SUCCESS: read_ok AND version_match + - name: Build success journal payload + when: (read_ok | bool) and (version_match | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}): + Banner='{{ banner_raw }}' Target='{{ expected_norm }}' + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_success + changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool) + + - name: Build payload to add indoor-update-success tag (DEV1) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + tag_add_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-update-success" + + - name: Publish indoor-update-success tag + when: tag_add_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_add_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_add_success + changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool) + + - name: Build payload to remove indoor-restart-scheduled tag (DEV1) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + tag_remove_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_remove" + task_result: "indoor-restart-scheduled" + + - name: Publish indoor-restart-scheduled tag removal + when: tag_remove_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_remove_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_remove_sched + changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) + + # --- Normalize firmware string and set custom field on success --- + - name: Capture raw firmware banner from DEV2 (for normalization) + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.set_fact: + fw_banner_raw: "{{ banner_raw }}" + + - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801') + when: + - journal_success_payload is defined + - (fw_banner_raw | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.set_fact: + fw_norm: >- + {{ + fw_banner_raw + if (fw_banner_raw | lower is search('-r[0-9]+$')) + else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) + }} + + - name: Debug normalized firmware (indoor_fwver) + when: fw_norm is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')" + + - name: Publish custom_field_set indoor_fwver + when: + - fw_norm is defined + - (fw_norm | length) > 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'indoor_fwver', + 'task_result': fw_norm + } | to_json }}" + payload_encoding: "string" + register: rmq_customfield_fw + changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool) + + # MISMATCH path: firmware readable but does NOT match expected target + - name: Build mismatch journal payload + when: (read_ok | bool) and (not (version_match | bool)) + delegate_to: localhost + ansible.builtin.set_fact: + journal_mismatch_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}): + Expected='{{ expected_norm }}' Got='{{ banner_raw }}' + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish mismatch journal to control queue + when: journal_mismatch_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_mismatch_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_mismatch + changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool) + + - name: Stop host after mismatch evaluation + when: journal_mismatch_payload is defined + ansible.builtin.meta: end_host + + # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart + - name: Build failure journal payload (indoor firmware read failed) + when: not (read_ok | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}): + fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}' + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish failure journal to control queue + when: journal_fail_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_fail + changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool) + + - name: Compute retry parameters (10 minutes) + when: not (read_ok | bool) + ansible.builtin.set_fact: + next_attempt: "{{ (attempt | int) + 1 }}" + next_delay_sec: 600 + next_delay_ms: "{{ 600000 }}" + + - name: Build final gave-up journal (max attempts reached) + when: (not (read_ok | bool)) and ((attempt | int) >= (effective_max_attempts | int)) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}. + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_gaveup + changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool) + + # Only schedule next attempt if we still have budget left + - name: Build delayed payload for next indoor attempt (10 min) + when: (not (read_ok | bool)) and ((attempt | int) < (effective_max_attempts | int)) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id | default('') }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + post_tasks: + - name: Cleanup tunnel and temp IP + block: + - ansible.builtin.debug: + msg: "Cleaning up tunnel + temp IP" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP from DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + failed_when: false + changed_when: false diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries b/files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries new file mode 100644 index 0000000..3051e9e --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_indoor_check.yml-bfr_3tries @@ -0,0 +1,201 @@ +--- +- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel) + hosts: all + gather_facts: no + + vars: + # ---------------- BusyBox-safe path prefix ---------------- + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # ---------------- DEV1 (outer device) ---------------- + dev1_user: "root" + dev1_pass: "wavewave" + dev1_iface: "br-wan" + + # ---------------- DEV2 (indoor behind DEV1) ---------------- + dev2_host: "192.168.1.1" + dev2_port: 22 + dev2_side_ip: "192.168.1.11/24" + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # ---------------- RabbitMQ (same env scheme as main playbooks) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + tasks: + # ---------------- Step 1: Read DEV1 hostname and sanity ---------------- + - name: Read DEV1 hostname + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 hostname mismatch + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + # ---------------- Step 2: Setup temporary IP for reachability ---------------- + - name: Add temporary IP on DEV1 (ignore if exists) + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 and + ('File exists' not in (add_ip.stdout | default(''))) and + ('File exists' not in (add_ip.stderr | default(''))) + + # ---------------- Step 3: Start SSH tunnel via DEV1 ---------------- + - name: Pick random free local port + delegate_to: localhost + ansible.builtin.shell: | + set -e + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n1)" + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi + done + exit 1 + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control dir + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + + - name: Start SSH ControlMaster tunnel via DEV1 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sshpass -p '{{ dev1_pass }}' ssh -f -N \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \ + "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + # ---------------- Step 4: Determine working password for DEV2 ---------------- + - name: Try both passfiles for DEV2 + delegate_to: localhost + ansible.builtin.shell: | + for f in {{ dev2_passfiles | join(' ') }}; do + if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then + echo "$f"; exit 0; + fi + done + echo NONE + register: dev2_passfile_try + changed_when: false + + - name: Save selected DEV2 passfile + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}" + changed_when: false + + - name: Stop if no valid passfile found + ansible.builtin.meta: end_host + when: dev2_passfile_used == "NONE" + + # ---------------- Step 5: Read firmware version on DEV2 ---------------- + - name: Read firmware version from DEV2 + delegate_to: localhost + ansible.builtin.shell: | + sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \ + root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown" + register: dev2_fwver + changed_when: false + ignore_errors: true + + - name: Show firmware version readout + delegate_to: localhost + ansible.builtin.debug: + msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" + + # ---------------- Step 6: Publish result journal ---------------- + - name: Build journal payload + delegate_to: localhost + ansible.builtin.set_fact: + journal_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}' + tunnel_port={{ _local_port }} + + - name: Publish journal to control queue + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_fwver + changed_when: (rmq_pub_fwver.json is defined) and (rmq_pub_fwver.json.routed | default(false) | bool) + + post_tasks: + - name: Cleanup tunnel and temp IP + block: + - ansible.builtin.debug: + msg: "Cleaning up tunnel + temp IP" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP from DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + failed_when: false + changed_when: false diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml-chat b/files/ansible-playbooks/afterupgrade_indoor_check.yml-chat new file mode 100644 index 0000000..e69de29 diff --git a/files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors b/files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors new file mode 100644 index 0000000..6797484 --- /dev/null +++ b/files/ansible-playbooks/afterupgrade_indoor_check.yml-fixingerrors @@ -0,0 +1,644 @@ +--- +- name: After-upgrade verification for indoor (DEV2 via DEV1 tunnel) + hosts: all + gather_facts: no + + vars: + # ---------------- BusyBox-safe path prefix ---------------- + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # ---------------- DEV1 (outer device) ---------------- + dev1_user: "root" + dev1_pass: "wavewave" + dev1_iface: "br-wan" + + # ---------------- DEV2 (indoor behind DEV1) ---------------- + dev2_host: "192.168.1.1" + dev2_port: 22 + dev2_side_ip: "192.168.1.11/24" + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # ---------------- RabbitMQ (same env scheme as main playbooks) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + tasks: + # ---- Normalize metadata (from delayed message) ---- + - name: Normalize after-upgrade metadata + delegate_to: localhost + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + target_version: "{{ target_version | default('') }}" + target_version_full: "{{ target_version | default('') }}" + + # ---- Controller-side TCP probe to DEV1 (no SSH to target yet) ---- + - name: Check if TCP/22 on DEV1 is reachable + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w5 {{ ansible_host | default(inventory_hostname) }} 22 + register: nc_probe + changed_when: false + ignore_errors: true + + # ---- If TCP down: journal + schedule next try or give up ---- + - name: Build failure journal (TCP unreachable) + mark retry + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check (attempt {{ attempt }}/{{ effective_max_attempts }}): + TCP 22 unreachable. Correlation={{ correlation_id }} + Original={{ original_emitted_at }} Target='{{ target_version_full }}' + _needs_retry: true + + - name: Publish failure journal (TCP unreachable) + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_tcp_fail + changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool) + + - name: Compute next-attempt delay (10 minutes) and counters + when: (_needs_retry | default(false)) | bool + delegate_to: localhost + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + next_delay_sec: 600 + next_delay_ms: 600000 + + - name: Build final gave-up journal (max attempts reached) + when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last error path=TCP. + Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}' + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - name: Stop host after final gave-up + when: journal_gaveup_payload is defined + ansible.builtin.meta: end_host + + - name: Build delayed after-upgrade payload for next attempt (10m) + when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id }}" + original_emitted_at: "{{ original_emitted_at }}" + target_version: "{{ target_version_full }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next attempt to holding (dead-letters to deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: Stop host after scheduling next attempt + when: (_needs_retry | default(false)) | bool + ansible.builtin.meta: end_host + + + # ---------------- Step 1: Read DEV1 hostname and sanity ---------------- + - name: Read DEV1 hostname + when: nc_probe.rc == 0 + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 hostname mismatch + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + # ---------------- Step 2: Setup temporary IP for reachability ---------------- + - name: Add temporary IP on DEV1 (ignore if exists) + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 and + ('File exists' not in (add_ip.stdout | default(''))) and + ('File exists' not in (add_ip.stderr | default(''))) + + # ---------------- Step 3: Start SSH tunnel via DEV1 ---------------- + - name: Pick random free local port + delegate_to: localhost + ansible.builtin.shell: | + set -e + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n1)" + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then echo "$p"; exit 0; fi + done + exit 1 + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control dir + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Start SSH ControlMaster tunnel via DEV1 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sshpass -p '{{ dev1_pass }}' ssh -f -N \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=15 \ + "{{ dev1_user }}@{{ ansible_host | default(inventory_hostname) }}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + # --- NEW: settle + check ControlMaster + TCP probe (prevents early passfile fail) --- + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running (ssh -O check) + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + # ---------------- Step 4: Determine working password for DEV2 ---------------- + - name: Try both passfiles for DEV2 + delegate_to: localhost + ansible.builtin.shell: | + for f in {{ dev2_passfiles | join(' ') }}; do + if sshpass -f "$f" ssh -p {{ _local_port }} -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 root@127.0.0.1 "echo OK" >/dev/null 2>&1; then + echo "$f"; exit 0; + fi + done + echo NONE + register: dev2_passfile_try + changed_when: false + + - name: Save selected DEV2 passfile + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "{{ (dev2_passfile_try.stdout | trim) }}" + changed_when: false + + - name: Debug selected DEV2 passfile + delegate_to: localhost + ansible.builtin.debug: + msg: "dev2_passfile_used={{ dev2_passfile_used }}" + + - name: Stop if no valid passfile found + ansible.builtin.meta: end_host + when: dev2_passfile_used == "NONE" + + # ---------------- Step 5: Read firmware version on DEV2 ---------------- + - name: Read firmware version from DEV2 + delegate_to: localhost + ansible.builtin.shell: | + sshpass -f "{{ dev2_passfile_used }}" ssh -p {{ _local_port }} \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout=10 \ + root@127.0.0.1 "cat /usr/lib/release/firmux 2>/dev/null || grep -i rev /etc/banner 2>/dev/null || echo unknown" + register: dev2_fwver + changed_when: false + ignore_errors: true + + - name: Show firmware version readout + delegate_to: localhost + ansible.builtin.debug: + msg: "Firmware version on DEV2: {{ (dev2_fwver.stdout | default('')) | trim }}" + + # ---------------- Retry metadata + success evaluation ---------------- + - name: Normalize retry metadata for indoor checker + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + effective_max_attempts: "{{ (max_attempts | default(3)) | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + + - name: Evaluate indoor success (firmware read OK) + delegate_to: localhost + ansible.builtin.set_fact: + indoor_success: "{{ (dev2_fwver.rc | default(1) == 0) + and ((dev2_fwver.stdout | default('') | trim) | length > 0) + and (not ((dev2_fwver.stdout | default('unknown') | lower) is search('unknown'))) }}" + + # ---------------- Step 6: Publish result journal (+ tags on success) ---------------- + # SUCCESS PATH: add tag, remove scheduled tag, and journal with "success" + - name: Build success journal payload + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check success: DEV2 firmware='{{ (dev2_fwver.stdout | default('unknown')) | trim }}' + tunnel_port={{ _local_port }} + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish success journal to control queue + when: journal_success_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_success + changed_when: (rmq_pub_success.json is defined) and (rmq_pub_success.json.routed | default(false) | bool) + + - name: Build payload to add indoor-update-success tag (DEV1) + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.set_fact: + tag_add_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-update-success" + + - name: Publish indoor-update-success tag + when: tag_add_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_add_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_add_success + changed_when: (rmq_tag_add_success.json is defined) and (rmq_tag_add_success.json.routed | default(false) | bool) + + - name: Build payload to remove indoor-restart-scheduled tag (DEV1) + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.set_fact: + tag_remove_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_remove" + task_result: "indoor-restart-scheduled" + + - name: Publish indoor-restart-scheduled tag removal + when: tag_remove_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_remove_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_remove_sched + changed_when: (rmq_tag_remove_sched.json is defined) and (rmq_tag_remove_sched.json.routed | default(false) | bool) + + # --- Normalize firmware string and set custom field on success --- + - name: Capture raw firmware banner from DEV2 (for normalization) + when: indoor_success | bool + delegate_to: localhost + ansible.builtin.set_fact: + fw_banner_raw: "{{ (dev2_fwver.stdout | default('')) | trim }}" + + - name: Normalize firmware string for indoor_fwver (e.g. '2.2.1 rev 6801' -> '2.2.1-r6801') + when: + - indoor_success | bool + - (fw_banner_raw | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.set_fact: + fw_norm: >- + {{ + fw_banner_raw + if (fw_banner_raw | lower is search('-r[0-9]+$')) + else (fw_banner_raw | regex_replace('\\s*[Rr][Ee][Vv]\\.?\\s*([0-9]+)\\s*$', '-r\\1')) + }} + + - name: Debug normalized firmware (indoor_fwver) + when: fw_norm is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "Normalized indoor_fwver={{ fw_norm }} (from='{{ fw_banner_raw }}')" + + - name: Publish custom_field_set indoor_fwver + when: + - indoor_success | bool + - fw_norm is defined + - (fw_norm | length) > 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'indoor_fwver', + 'task_result': fw_norm + } | to_json }}" + payload_encoding: "string" + register: rmq_customfield_fw + changed_when: (rmq_customfield_fw.json is defined) and (rmq_customfield_fw.json.routed | default(false) | bool) + + # FAILURE / RETRY PATH: journal + schedule next attempt (up to 3 total), 10 minutes apart + - name: Build failure journal payload (indoor firmware read failed) + when: not (indoor_success | bool) + delegate_to: localhost + ansible.builtin.set_fact: + journal_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check FAILED (attempt {{ attempt }}/{{ effective_max_attempts }}): + fwread_rc={{ dev2_fwver.rc | default('NA') }}, output='{{ (dev2_fwver.stdout | default('') | trim) }}' + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish failure journal to control queue + when: journal_fail_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_fail + changed_when: (rmq_pub_fail.json is defined) and (rmq_pub_fail.json.routed | default(false) | bool) + + - name: Compute retry parameters (10 minutes) + when: not (indoor_success | bool) + ansible.builtin.set_fact: + next_attempt: "{{ (attempt | int) + 1 }}" + next_delay_sec: 600 + next_delay_ms: "{{ 600000 }}" + + - name: Build final gave-up journal (max attempts reached) + when: (not (indoor_success | bool)) and ((attempt | int) >= (effective_max_attempts | int)) + delegate_to: localhost + ansible.builtin.set_fact: + journal_gaveup_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + afterupgrade_indoor_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}): + Exhausted attempts. Last fwread_rc={{ dev2_fwver.rc | default('NA') }}. + Correlation={{ correlation_id | default('') }} Original={{ original_emitted_at | default('') }} + + - name: Publish final gave-up journal + when: journal_gaveup_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_gaveup_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_gaveup + changed_when: (rmq_pub_gaveup.json is defined) and (rmq_pub_gaveup.json.routed | default(false) | bool) + + # Only schedule next attempt if we still have budget left + - name: Build delayed payload for next indoor attempt (10 min) + when: (not (indoor_success | bool)) and ((attempt | int) < (effective_max_attempts | int)) + delegate_to: localhost + ansible.builtin.set_fact: + delayed_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + attempt: "{{ next_attempt | int }}" + max_attempts: "{{ effective_max_attempts | int }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default('') }}" + current_delay_sec: "{{ next_delay_sec | int }}" + schema_version: 1 + + - name: Publish delayed next indoor attempt (holding + TTL → deviceconfig) + when: delayed_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + expiration: "{{ (next_delay_ms | int) | string }}" + correlation_id: "{{ correlation_id | default('') }}" + routing_key: "deviceconfig" + payload: "{{ delayed_payload | to_json }}" + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + post_tasks: + - name: Cleanup tunnel and temp IP + block: + - ansible.builtin.debug: + msg: "Cleaning up tunnel + temp IP" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP from DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + failed_when: false + changed_when: false + diff --git a/files/ansible-playbooks/basicpass b/files/ansible-playbooks/basicpass new file mode 100755 index 0000000..277d8b9 --- /dev/null +++ b/files/ansible-playbooks/basicpass @@ -0,0 +1 @@ +wavewave diff --git a/files/ansible-playbooks/basicpass2 b/files/ansible-playbooks/basicpass2 new file mode 100755 index 0000000..7fbe952 --- /dev/null +++ b/files/ansible-playbooks/basicpass2 @@ -0,0 +1 @@ +admin diff --git a/files/ansible-playbooks/checkversioncli.yml b/files/ansible-playbooks/checkversioncli.yml new file mode 100644 index 0000000..99c96ab --- /dev/null +++ b/files/ansible-playbooks/checkversioncli.yml @@ -0,0 +1,18 @@ +--- +- name: Simple script to show current version + hosts: all + gather_facts: no + + tasks: + - name: grab current version + ansible.builtin.raw: | + set -e + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + cat /etc/banner | grep 2 + register: banner_out + changed_when: true + + - name: show the current version + ansible.builtin.debug: + msg: "{{ banner_out.stdout | trim }}" + diff --git a/files/ansible-playbooks/deploy-scroll24.yml b/files/ansible-playbooks/deploy-scroll24.yml new file mode 100644 index 0000000..0a8a58c --- /dev/null +++ b/files/ansible-playbooks/deploy-scroll24.yml @@ -0,0 +1,277 @@ +--- +- name: Deploy scroll24 script and cron configuration + hosts: all + gather_facts: no + + vars: + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # Version tag for scroll24 deployment (operator-controlled) + scroll24_version: "v1.1" + + # Controller-side source paths (same place as wifidebug.sh) + src_script: "files/scroll24.sh" + src_cron_snippet: "files/crond-root-scroll24" + + # Remote destinations + remote_script: "/root/scroll24.sh" + remote_crontab: "/etc/crontabs/root" + tmp_cron_snippet: "/tmp/crond-root-scroll24.snippet" + tmp_cron_new: "/tmp/cron.root.new" + crontab_backup_dir: "/etc/crontabs" + + # RabbitMQ (same contract/style as your other playbooks) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + tasks: + + ########################################################################## + # 0. Controller sanity check for tools + ########################################################################## + - name: Verify controller tools (sshpass, sha256sum/busybox) + delegate_to: localhost + shell: | + command -v sshpass >/dev/null 2>&1 \ + && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1) + args: { executable: /bin/bash } + register: ctrl_tools + changed_when: false + failed_when: ctrl_tools.rc != 0 + + ########################################################################## + # 1. Upload /root/scroll24.sh if changed (sha256 verified) + ########################################################################## + - name: Compute local sha256 of scroll24.sh + delegate_to: localhost + command: sha256sum {{ src_script }} + register: sha_local + changed_when: false + + - name: Extract local sha256 digest (regex) + delegate_to: localhost + set_fact: + local_hash: "{{ (sha_local.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}" + + - name: Compute remote sha256 of /root/scroll24.sh + raw: "sha256sum {{ remote_script }} 2>/dev/null || busybox sha256sum {{ remote_script }} 2>/dev/null || true" + register: sha_remote + changed_when: false + failed_when: false + + - name: Extract remote sha256 digest (regex) + set_fact: + remote_hash: "{{ (sha_remote.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}" + + - name: Decide if scroll24.sh needs upload + set_fact: + script_changed: "{{ (remote_hash | length == 0) or (local_hash != remote_hash) }}" + + - name: Upload scroll24.sh via scp (if needed) + when: script_changed | bool + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ src_script }} + {{ ssh_user }}@{{ ansible_host }}:{{ remote_script }} + register: scp_scroll24 + changed_when: true + + - name: Ensure /root/scroll24.sh permissions and ownership + raw: "chown root:root {{ remote_script }} && chmod 0755 {{ remote_script }}" + changed_when: script_changed | bool + + - name: Recompute remote sha256 after upload + when: script_changed | bool + raw: "sha256sum {{ remote_script }} || busybox sha256sum {{ remote_script }}" + register: sha_remote_after + changed_when: false + failed_when: false + + - name: Extract remote-after sha256 digest (regex) + when: script_changed | bool + set_fact: + remote_hash_after: "{{ (sha_remote_after.stdout | default('')) | regex_search('([A-Fa-f0-9]{64})') | default('') }}" + + - name: Fail if scroll24.sh sha256 mismatch after upload + when: script_changed | bool and (remote_hash_after | default('')) != (local_hash | default('')) + fail: + msg: "sha256 mismatch between controller and remote scroll24.sh" + + ########################################################################## + # 1b. Extract the period (minutes) from the first line of the snippet + ########################################################################## + - name: Extract period minutes from first snippet line (*/N …) + delegate_to: localhost + shell: | + awk 'NR==1{ + f=$1; + if (f ~ /^\*\/[0-9]+$/) { gsub("^\\*/","",f); print f; exit } + else if (f ~ /^[0-9]+$/) { print f; exit } + else { print ""; exit } + }' {{ src_cron_snippet }} + args: { executable: /bin/bash } + register: cron_period_cmd + changed_when: false + failed_when: false + + - name: Set cron_period fact + delegate_to: localhost + set_fact: + cron_period: "{{ (cron_period_cmd.stdout | trim) }}" + + ########################################################################## + # 2. Upload cron snippet to device /tmp (operator-provided content) + ########################################################################## + - name: Push cron snippet to device tmp + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ src_cron_snippet }} + {{ ssh_user }}@{{ ansible_host }}:{{ tmp_cron_snippet }} + register: scp_snippet + changed_when: true + + ########################################################################## + # 3. Crontab subset logic (only scroll24.sh lines) + ########################################################################## + - name: Ensure /etc/crontabs/root exists (with perms) + raw: | + if [ ! -f {{ remote_crontab }} ]; then + touch {{ remote_crontab }}; + fi + chown root:root {{ remote_crontab }}; + chmod 0644 {{ remote_crontab }}; + changed_when: false + + - name: Extract current scroll24 lines from crontab + raw: "grep -F 'scroll24.sh' {{ remote_crontab }} || true" + register: cron_subset + changed_when: false + failed_when: false + + - name: Normalize controller snippet (for compare only) + delegate_to: localhost + shell: | + grep -v '^[[:space:]]*$' {{ src_cron_snippet }} \ + | sed 's/[[:space:]]\+/ /g' | sed 's/[[:space:]]*$//' | sort -u + register: norm_snippet + changed_when: false + + - name: Normalize device scroll24 subset (for compare only) + delegate_to: localhost + shell: | + printf "%s\n" "{{ cron_subset.stdout | default('') }}" \ + | grep -v '^[[:space:]]*$' \ + | sed 's/[[:space:]]\+/ /g' | sed 's/[[:space:]]*$//' | sort -u + register: norm_remote_subset + changed_when: false + + - name: Decide if crontab needs update (subset compare) + delegate_to: localhost + set_fact: + crontab_changed: "{{ (norm_snippet.stdout | trim) != (norm_remote_subset.stdout | trim) }}" + + - name: Backup current crontab (timestamped) + when: crontab_changed | bool + raw: "cp -a {{ remote_crontab }} {{ crontab_backup_dir }}/root.bak.$(date +%Y%m%d%H%M%S)" + changed_when: true + + - name: Replace scroll24 lines in crontab (preserve all others; tidy splice) + when: crontab_changed | bool + raw: "grep -v 'scroll24\\.sh' {{ remote_crontab }} > {{ tmp_cron_new }} && awk 'BEGIN{for(i=1;i<=NR;i++)a[i]=$0} {a[NR]=$0} END{e=NR; while(e>0 && a[e] ~ /^[[:space:]]*$/){e--}; for(i=1;i<=e;i++) print a[i]}' {{ tmp_cron_new }} > {{ tmp_cron_new }}.trim && mv {{ tmp_cron_new }}.trim {{ tmp_cron_new }} && cat {{ tmp_cron_snippet }} >> {{ tmp_cron_new }} && printf '\\n' >> {{ tmp_cron_new }} && mv {{ tmp_cron_new }} {{ remote_crontab }} && chown root:root {{ remote_crontab }} && chmod 0644 {{ remote_crontab }}" + changed_when: true + + ########################################################################## + # 4. Restart crond if script or cron changed (with :51–:59 guard) + ########################################################################## + - name: Check if restart required + set_fact: + need_restart: "{{ (script_changed | bool) or (crontab_changed | bool) }}" + + - name: Get current seconds + when: need_restart | bool + raw: "date +%S" + register: nowsec + changed_when: false + + - name: Sleep 10s if seconds 51-59 + when: need_restart | bool and (nowsec.stdout | int >= 51) + pause: + seconds: 10 + + - name: Restart crond via move/move + when: need_restart | bool + raw: "mv /tmp/launchd/services/crond /root/crond && sleep 1 && mv /root/crond /tmp/launchd/services/crond" + register: crond_restart + changed_when: true + failed_when: false + + - name: Verify crond is running + when: need_restart | bool + raw: "pgrep -f '/usr/sbin/crond' || busybox pgrep crond || echo missing" + register: crond_pid + changed_when: false + failed_when: false + + ########################################################################## + # 5. Final journals to RabbitMQ (controller) + ########################################################################## + - name: Publish final scroll24 deployment journal + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'scroll24: ' + ~ (script_changed | ternary('script installed/updated; ', 'script up-to-date; ')) + ~ (crontab_changed | ternary('cron updated; ', 'cron already matches; ')) + ~ (need_restart | ternary('crond restarted; ', 'crond unchanged; ')) + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Set NetBox custom field scroll24 -> "_" + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'scroll24', + 'task_result': scroll24_version ~ '_' ~ (cron_period | default('')) + } | to_json }}" + payload_encoding: "string" + changed_when: false + diff --git a/files/ansible-playbooks/full-upgrade-check.yml b/files/ansible-playbooks/full-upgrade-check.yml new file mode 100644 index 0000000..7a40fa6 --- /dev/null +++ b/files/ansible-playbooks/full-upgrade-check.yml @@ -0,0 +1,20 @@ +- import_playbook: update-rebootin222.yml + vars: + rebootin: 0 + +- name: + hosts: all # will match your limited hosts + gather_facts: false + tags: always + tasks: + - name: Wait 5 minutes + ansible.builtin.wait_for: + timeout: "{{ pause_minutes | default(300) }}" + run_once: true + delegate_to: localhost + check_mode: no + tags: always + +- import_playbook: checkversioncli.yml +#- import_playbook: setntptime.yml + diff --git a/files/ansible-playbooks/full-upgrade-wifi14.yml b/files/ansible-playbooks/full-upgrade-wifi14.yml new file mode 100644 index 0000000..515b66d --- /dev/null +++ b/files/ansible-playbooks/full-upgrade-wifi14.yml @@ -0,0 +1,21 @@ +- import_playbook: update-rebootin.yml + vars: + rebootin: 0 + +- name: + hosts: all # will match your limited hosts + gather_facts: false + tags: always + tasks: + - name: Wait 5 minutes + ansible.builtin.wait_for: + timeout: "{{ pause_minutes | default(300) }}" + run_once: true + delegate_to: localhost + check_mode: no + tags: always + +- import_playbook: checkversioncli.yml +- import_playbook: setntptime.yml +- import_playbook: wifidebug14.yml +- import_playbook: system-stop-system-start.yml diff --git a/files/ansible-playbooks/multissidfix-changessid.yml b/files/ansible-playbooks/multissidfix-changessid.yml new file mode 100644 index 0000000..f754125 --- /dev/null +++ b/files/ansible-playbooks/multissidfix-changessid.yml @@ -0,0 +1,826 @@ +--- +- name: "Multi-SSID fix (phase 1: persist + runtime apply)" + hosts: all + gather_facts: false + + vars: + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (same contract as wifidebug16.yml) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + # Reboot parachute delay (seconds) + parachute_delay_s: 600 + + # Local working paths (controller) + local_cfg: "/tmp/{{ inventory_hostname }}_config.json" + local_new: "/tmp/{{ inventory_hostname }}_config.json.new" + local_step: "/tmp/{{ inventory_hostname }}_config.step" + local_diff: "/tmp/{{ inventory_hostname }}_config.diff" + local_dd: "/tmp/{{ inventory_hostname }}_dd.json" + + # External scripts/filters (next to wifidebug.sh in your repo) + dd_script_src: "files/wirelessduediligence.sh" + f_1vap0_src: "files/filter_1vap_id0.jq" + f_1vap1_src: "files/filter_1vap_id1.jq" + f_2_01_src: "files/filter_2vaps_0_1.jq" + f_2_10_src: "files/filter_2vaps_1_0.jq" + + # Controller tmp destinations + dd_script: "/tmp/wirelessduediligence.sh" + f_1vap0: "/tmp/filter_1vap_id0.jq" + f_1vap1: "/tmp/filter_1vap_id1.jq" + f_2_01: "/tmp/filter_2vaps_0_1.jq" + f_2_10: "/tmp/filter_2vaps_1_0.jq" + + # Remote (device) + remote_cfg: "/tmp/config.json" + remote_new: "/tmp/config.json.new" + remote_backup: "/tmp/config.json.multissidfix1.backup" + + # Optional new SSID name (set via env NEW_SSID to trigger rename) + new_ssid_name: "{{ lookup('env','NEW_SSID') | default('ikeja R5 a day hotspot', true) }}" + + tasks: + + ########################################################################## + # a) Connectivity + basic sanity + ########################################################################## + - name: SSH reachability probe + raw: "echo ping" + register: ping_result + ignore_errors: true + + - name: Stop if SSH unreachable (soft-fail) + when: ping_result is failed + block: + - name: Journal soft-fail (no SSH) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail — SSH not reachable' + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + - name: Ensure remote /tmp/config.json exists + raw: "test -s {{ remote_cfg }}" + register: remote_cfg_check + changed_when: false + + - name: Stop if /tmp/config.json missing (soft-fail) + when: remote_cfg_check.rc != 0 + block: + - name: Journal soft-fail (missing config.json) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail — missing /tmp/config.json' + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + - name: Check controller tools (jq/sshpass/diff/sha256sum) + delegate_to: localhost + shell: | + command -v jq >/dev/null 2>&1 \ + && command -v sshpass >/dev/null 2>&1 \ + && command -v diff >/dev/null 2>&1 \ + && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1) + args: { executable: /bin/bash } + register: ctrl_tools + changed_when: false + + - name: Stop if controller tools missing (soft-fail) + when: ctrl_tools.rc != 0 + block: + - name: Journal soft-fail (missing tools) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail — controller missing jq/sshpass/diff/sha256sum' + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + ########################################################################## + # b) Due diligence (external script, run on controller) + ########################################################################## + - name: Copy due diligence script to controller tmp + delegate_to: localhost + copy: + src: "{{ dd_script_src }}" + dest: "{{ dd_script }}" + mode: "0755" + + - name: Pull /tmp/config.json from device to controller + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:{{ remote_cfg }} + {{ local_cfg }} + + - name: Run due diligence on controller + delegate_to: localhost + shell: "sh {{ dd_script | quote }} {{ local_cfg | quote }} > {{ local_dd | quote }}" + args: { executable: /bin/sh } + changed_when: false + + - name: Parse due diligence JSON + delegate_to: localhost + set_fact: + dd: "{{ lookup('file', local_dd) | from_json }}" + + # Normalize types for robust `when:` checks + - name: Extract & normalize dd facts + delegate_to: localhost + set_fact: + w0_total_i: "{{ (dd.radios.wifi0.vaps_total | int) }}" + w1_total_i: "{{ (dd.radios.wifi1.vaps_total | int) }}" + w0_idx_i: "{{ (dd.radios.wifi0.indices | map('int') | list) }}" + w1_idx_i: "{{ (dd.radios.wifi1.indices | map('int') | list) }}" + + ########################################################################## + # c) Soft-fail: layout policy checks (one journal) + ########################################################################## + - name: Init soft-fail reasons (policy) + delegate_to: localhost + set_fact: + softfail_reasons: [] + + - name: wifi0 — add reason if ONE_VAP but index not 0/1 + delegate_to: localhost + when: (w0_total_i | int) == 1 and (0 not in w0_idx_i and 1 not in w0_idx_i) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi0 ONE_VAP but index not 0/1 (indices=' ~ (w0_idx_i | string) ~ ')' ] }}" + + - name: wifi0 — add reason if TWO_VAPS but indices not (0,1)/(1,0) + delegate_to: localhost + when: (w0_total_i | int) == 2 and (w0_idx_i | sort not in [[0,1],[1,0]]) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi0 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w0_idx_i | string) ~ ')' ] }}" + + - name: wifi0 — add reason if MORE_THAN_TWO_VAPS + delegate_to: localhost + when: (w0_total_i | int) > 2 + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi0 MORE_THAN_TWO_VAPS (indices=' ~ (w0_idx_i | string) ~ ')' ] }}" + + - name: wifi1 — add reason if ONE_VAP but index not 0/1 + delegate_to: localhost + when: (w1_total_i | int) == 1 and (0 not in w1_idx_i and 1 not in w1_idx_i) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi1 ONE_VAP but index not 0/1 (indices=' ~ (w1_idx_i | string) ~ ')' ] }}" + + - name: wifi1 — add reason if TWO_VAPS but indices not (0,1)/(1,0) + delegate_to: localhost + when: (w1_total_i | int) == 2 and (w1_idx_i | sort not in [[0,1],[1,0]]) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi1 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w1_idx_i | string) ~ ')' ] }}" + + - name: wifi1 — add reason if MORE_THAN_TWO_VAPS + delegate_to: localhost + when: (w1_total_i | int) > 2 + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi1 MORE_THAN_TWO_VAPS (indices=' ~ (w1_idx_i | string) ~ ')' ] }}" + + - name: Publish soft-fail journal & stop host (policy) + when: (softfail_reasons | length) > 0 + block: + - name: Journal soft-fail (policy) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail (policy) — ' ~ (softfail_reasons | join('; ')) + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + ########################################################################## + # d) Filters to controller tmp + candidate generation (per radio) + ########################################################################## + - name: Copy jq filters to controller tmp + delegate_to: localhost + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + loop: + - { src: "{{ f_1vap0_src }}", dest: "{{ f_1vap0 }}" } + - { src: "{{ f_1vap1_src }}", dest: "{{ f_1vap1 }}" } + - { src: "{{ f_2_01_src }}", dest: "{{ f_2_01 }}" } + - { src: "{{ f_2_10_src }}", dest: "{{ f_2_10 }}" } + + - name: Start from current config as working file + delegate_to: localhost + copy: + src: "{{ local_cfg }}" + dest: "{{ local_step }}" + mode: "0644" + + # wifi0 choice + - name: wifi0 — apply filter 1vap_id0 + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 1 and (0 in w0_idx_i) + changed_when: true + + - name: wifi0 — apply filter 1vap_id1 (Policy B) + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 1 and (1 in w0_idx_i) + changed_when: true + + - name: wifi0 — apply filter 2vaps_0_1 + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 2 and (w0_idx_i | sort == [0,1]) + changed_when: true + + - name: wifi0 — apply filter 2vaps_1_0 (actual order 1,0) + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 2 and (w0_idx_i == [1,0]) + changed_when: true + + # wifi1 choice + - name: wifi1 — apply filter 1vap_id0 + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 1 and (0 in w1_idx_i) + changed_when: true + + - name: wifi1 — apply filter 1vap_id1 (Policy B) + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 1 and (1 in w1_idx_i) + changed_when: true + + - name: wifi1 — apply filter 2vaps_0_1 + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 2 and (w1_idx_i | sort == [0,1]) + changed_when: true + + - name: wifi1 — apply filter 2vaps_1_0 (actual order 1,0) + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 2 and (w1_idx_i == [1,0]) + changed_when: true + + # Optional SSID rename (only if new_ssid_name is defined) + - name: Set SSID for the single enabled AP VAP per radio (wifi0 & wifi1) + delegate_to: localhost + shell: | + jq --arg SSID "{{ new_ssid_name }}" ' + .wireless.radios |= + with_entries( + .value.vaps = + ((.value.vaps // []) | map( + if (.mode=="ap" and (.enabled==true)) then + (.ssid = $SSID) + else . + end + )) + ) + ' {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }} + args: { executable: /bin/bash } + when: new_ssid_name is defined + changed_when: true + + - name: Debug normalized types + delegate_to: localhost + debug: + msg: + - "w0_total_i(type)={{ w0_total_i | type_debug }} value={{ w0_total_i }}" + - "w1_total_i(type)={{ w1_total_i | type_debug }} value={{ w1_total_i }}" + + - name: Move working file to final candidate + delegate_to: localhost + copy: + src: "{{ local_step }}" + dest: "{{ local_new }}" + mode: "0644" + + ########################################################################## + # e) Candidate checks — accumulate reasons; soft-fail once if any + ########################################################################## + - name: Init soft-fail reasons (candidate) + delegate_to: localhost + set_fact: + cand_reasons: [] + + - name: Check JSON validity + delegate_to: localhost + shell: "jq -e '.' {{ local_new | quote }} >/dev/null" + args: { executable: /bin/bash } + register: json_valid + changed_when: false + failed_when: false + + - name: Add reason if JSON invalid + delegate_to: localhost + when: json_valid.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'invalid JSON syntax in candidate' ] }}" + + - name: Semantic — wifi0 exactly one enabled AP VAP + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w0 + changed_when: false + failed_when: false + + - name: Add reason if wifi0 semantic fails + delegate_to: localhost + when: sem_w0.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi0 must have exactly one enabled AP VAP' ] }}" + + - name: Semantic — wifi1 exactly one enabled AP VAP + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w1 + changed_when: false + failed_when: false + + - name: Add reason if wifi1 semantic fails + delegate_to: localhost + when: sem_w1.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi1 must have exactly one enabled AP VAP' ] }}" + + - name: Semantic — wifi0 enabled AP VAP has lbd=true + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w0_lbd + changed_when: false + failed_when: false + + - name: Add reason if wifi0 lbd check fails + delegate_to: localhost + when: sem_w0_lbd.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi0 enabled AP VAP must have lbd=true' ] }}" + + - name: Semantic — wifi1 enabled AP VAP has lbd=true + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w1_lbd + changed_when: false + failed_when: false + + - name: Add reason if wifi1 lbd check fails + delegate_to: localhost + when: sem_w1_lbd.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi1 enabled AP VAP must have lbd=true' ] }}" + + - name: Get original file size + delegate_to: localhost + stat: + path: "{{ local_cfg }}" + register: stat_old + changed_when: false + + - name: Get new file size + delegate_to: localhost + stat: + path: "{{ local_new }}" + register: stat_new + changed_when: false + + - name: Add reason if size delta > 2% + delegate_to: localhost + when: stat_old.stat.size | int == 0 or + ( ((stat_new.stat.size | int) - (stat_old.stat.size | int)) | abs ) > ( (stat_old.stat.size | int) * 0.02 ) + set_fact: + cand_reasons: "{{ cand_reasons + [ 'candidate size delta >2% (old=' ~ (stat_old.stat.size | string) ~ ', new=' ~ (stat_new.stat.size | string) ~ ')' ] }}" + + - name: Build unified diff (first 200 lines) + delegate_to: localhost + shell: "diff -u {{ local_cfg | quote }} {{ local_new | quote }} | sed -n '1,200p' > {{ local_diff | quote }} || true" + args: { executable: /bin/bash } + changed_when: false + + - name: Show unified diff (first 200 lines) + delegate_to: localhost + debug: + msg: "{{ lookup('file', local_diff) | default('(no diff output)') }}" + + - name: sha256 (controller) of candidate + delegate_to: localhost + command: sha256sum {{ local_new }} + register: sha_local + changed_when: false + + - name: Copy candidate to device temp + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ local_new }} + {{ ssh_user }}@{{ ansible_host }}:{{ remote_new }} + register: scp_new + changed_when: true + + - name: sha256 (remote) of candidate + raw: "sha256sum {{ remote_new }} || busybox sha256sum {{ remote_new }}" + register: sha_remote + changed_when: false + + - name: Add reason if sha256 mismatch + delegate_to: localhost + when: (sha_local.stdout.split()[0]) != (sha_remote.stdout.split()[0]) + set_fact: + cand_reasons: "{{ cand_reasons + [ 'sha256 mismatch controller vs remote' ] }}" + + - name: Detect SSIDs with NBSP (informational) + delegate_to: localhost + shell: | + nbsp="$(printf '\302\240')" + jq -r '..|objects|.ssid? // empty' {{ local_new | quote }} | grep -F "$nbsp" || true + args: { executable: /bin/bash } + register: nbsp_lines + changed_when: false + + - name: Save NBSP report lines + set_fact: + ssid_nbsp_lines: "{{ nbsp_lines.stdout_lines | default([]) }}" + + - name: Publish soft-fail journal & stop host (candidate issues) + when: (cand_reasons | length) > 0 + block: + - name: Journal soft-fail (candidate) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail (candidate) — ' ~ (cand_reasons | join('; ')) + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + ########################################################################## + # f) Backup, parachute, promote, persist, apply, outcome + ########################################################################## + - name: Backup current config on device + raw: "cp -a {{ remote_cfg }} {{ remote_backup }}" + changed_when: true + + # --- PRE-APPLY HEADS-UP --- + - name: Journal — candidate validated, will promote/persist/apply shortly + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'multissidfix: candidate passed checks; proceeding to backup + parachute + promote + sysconf -w + system-stop/start. ' + ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Arm reboot parachute (BusyBox reboot -d {{ parachute_delay_s }}) + raw: "sh -c 'trap \"\" HUP; reboot -d {{ parachute_delay_s }} /dev/null 2>&1 &'" + changed_when: true + ignore_errors: true + + - name: Promote candidate to active config + raw: "mv {{ remote_new }} {{ remote_cfg }} && chown root:root {{ remote_cfg }} && chmod 0644 {{ remote_cfg }}" + changed_when: true + + # IMPORTANT: persist BEFORE runtime restart + - name: Persist config to flash (sysconf -w) + raw: "sysconf -w" + register: sysconf_write + changed_when: true + failed_when: false + + # --- PRE-RESTART COMMIT --- + - name: Pre-restart journal (promoted & persisted; about to restart from controller) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'multissidfix: promoted candidate, persisted (sysconf -w rc=' ~ (sysconf_write.rc | default('n/a')) | string ~ '), ' + ~ 'system-stop/start launching from controller with 10s cap; ' + ~ 'parachute -d ' ~ parachute_delay_s|string ~ 's armed. ' + ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string + ~ ', nbsp_ssids=' ~ (ssid_nbsp_lines|length)|string + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Launch restart from controller with 10s cap (SSH command) + delegate_to: localhost + shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'" + args: + executable: /bin/bash + register: restart_kick + changed_when: true + failed_when: false + + - name: Note restart_kick result (debug) + delegate_to: localhost + debug: + msg: + - "restart_kick.rc={{ restart_kick.rc }}" + - "stdout(last 5 lines): {{ (restart_kick.stdout_lines | default([]))[-5:] | default([]) }}" + - "stderr(last 5 lines): {{ (restart_kick.stderr_lines | default([]))[-5:] | default([]) }}" + + # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3) + - name: Probe SSH with nc (24 tries, 5s each) + delegate_to: localhost + shell: "nc -z -w 3 {{ ansible_host }} 22" + register: nc_probe + retries: 24 + delay: 5 + until: nc_probe.rc == 0 + changed_when: false + failed_when: false + + - name: Set ssh_up fact + delegate_to: localhost + set_fact: + ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}" + + # --- SUCCESS PATH --- + - name: Post-return actions (only if SSH came back) + when: ssh_up | bool + block: + + - name: Disarm delayed reboot if present + raw: "pgrep -x reboot && kill -9 $(pgrep -x reboot) || true" + changed_when: true + ignore_errors: true + register: disarm_reboot + + # Re-run simple semantic checks on the active config + - name: Device semantic — wifi0 exactly one enabled AP VAP + raw: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w0 + changed_when: false + failed_when: false + + - name: Device semantic — wifi1 exactly one enabled AP VAP + raw: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w1 + changed_when: false + failed_when: false + + - name: Device semantic — wifi0 enabled AP VAP has lbd=true + raw: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w0_lbd + changed_when: false + failed_when: false + + - name: Device semantic — wifi1 enabled AP VAP has lbd=true + raw: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w1_lbd + changed_when: false + failed_when: false + + - name: sha256 (remote) of current active config (post-return) + raw: "sha256sum {{ remote_cfg }} || busybox sha256sum {{ remote_cfg }}" + register: sha_remote_after + changed_when: false + failed_when: false + + - name: Decide post-return checks summary (controller side) + delegate_to: localhost + set_fact: + multissid_post_summary: + reboot_disarmed: "{{ (disarm_reboot is defined) and (disarm_reboot.rc is defined) and (disarm_reboot.rc in [0]) }}" + persisted_ok: "{{ (sysconf_write.rc | default(0)) == 0 }}" + dev_sem_ok: + w0_one: "{{ (dev_sem_w0.rc | default(1)) == 0 }}" + w1_one: "{{ (dev_sem_w1.rc | default(1)) == 0 }}" + w0_lbd: "{{ (dev_sem_w0_lbd.rc | default(1)) == 0 }}" + w1_lbd: "{{ (dev_sem_w1_lbd.rc | default(1)) == 0 }}" + cfg_hash_match: "{{ (sha_remote_after.stdout.split()[0] | default('')) == (sha_remote.stdout.split()[0] | default('')) }}" + old_size: "{{ stat_old.stat.size | default('n/a') }}" + new_size: "{{ stat_new.stat.size | default('n/a') }}" + nbsp_count: "{{ (ssid_nbsp_lines | default([])) | length }}" + + - name: Publish final success journal (device returned; persisted; checks pass) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': + ( + 'multissidfix: success — device is back; ' + ~ (multissid_post_summary.reboot_disarmed | ternary('parachute disarmed; ', 'parachute was not armed; ')) + ~ (multissid_post_summary.persisted_ok | ternary('persisted (sysconf -w); ', 'persist failed; ')) + ~ 'semantics: ' + ~ 'w0_one=' ~ (multissid_post_summary.dev_sem_ok.w0_one | string) ~ ', ' + ~ 'w1_one=' ~ (multissid_post_summary.dev_sem_ok.w1_one | string) ~ ', ' + ~ 'w0_lbd=' ~ (multissid_post_summary.dev_sem_ok.w0_lbd | string) ~ ', ' + ~ 'w1_lbd=' ~ (multissid_post_summary.dev_sem_ok.w1_lbd | string) ~ '; ' + ~ 'cfg_match=' ~ (multissid_post_summary.cfg_hash_match | string) ~ '; ' + ~ 'size(old/new)=' ~ (multissid_post_summary.old_size | string) ~ '/' ~ (multissid_post_summary.new_size | string) ~ '; ' + ~ 'nbsp_ssids=' ~ (multissid_post_summary.nbsp_count | string) + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Set NetBox custom field multissidfix=v1.1 + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'multissidfix', + 'task_result': 'v1.1' + } | to_json }}" + payload_encoding: "string" + changed_when: false + + + # --- FAILURE PATH (device did not return) --- + - name: Journal — restart/SSH probe failed + when: not ssh_up + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'multissidfix: restart issued; SSH did not return after 24 x 5s checks — leaving parachute active.' + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Stop host after failed SSH probe + when: not ssh_up + meta: end_host + diff --git a/files/ansible-playbooks/multissidfix.yml b/files/ansible-playbooks/multissidfix.yml new file mode 100644 index 0000000..7716fdc --- /dev/null +++ b/files/ansible-playbooks/multissidfix.yml @@ -0,0 +1,802 @@ +--- +- name: "Multi-SSID fix (phase 1: persist + runtime apply)" + hosts: all + gather_facts: false + + vars: + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (same contract as wifidebug16.yml) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + # Reboot parachute delay (seconds) + parachute_delay_s: 600 + + # Local working paths (controller) + local_cfg: "/tmp/{{ inventory_hostname }}_config.json" + local_new: "/tmp/{{ inventory_hostname }}_config.json.new" + local_step: "/tmp/{{ inventory_hostname }}_config.step" + local_diff: "/tmp/{{ inventory_hostname }}_config.diff" + local_dd: "/tmp/{{ inventory_hostname }}_dd.json" + + # External scripts/filters (next to wifidebug.sh in your repo) + dd_script_src: "files/wirelessduediligence.sh" + f_1vap0_src: "files/filter_1vap_id0.jq" + f_1vap1_src: "files/filter_1vap_id1.jq" + f_2_01_src: "files/filter_2vaps_0_1.jq" + f_2_10_src: "files/filter_2vaps_1_0.jq" + + # Controller tmp destinations + dd_script: "/tmp/wirelessduediligence.sh" + f_1vap0: "/tmp/filter_1vap_id0.jq" + f_1vap1: "/tmp/filter_1vap_id1.jq" + f_2_01: "/tmp/filter_2vaps_0_1.jq" + f_2_10: "/tmp/filter_2vaps_1_0.jq" + + # Remote (device) + remote_cfg: "/tmp/config.json" + remote_new: "/tmp/config.json.new" + remote_backup: "/tmp/config.json.multissidfix1.backup" + + tasks: + + ########################################################################## + # a) Connectivity + basic sanity + ########################################################################## + - name: SSH reachability probe + raw: "echo ping" + register: ping_result + ignore_errors: true + + - name: Stop if SSH unreachable (soft-fail) + when: ping_result is failed + block: + - name: Journal soft-fail (no SSH) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail — SSH not reachable' + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + - name: Ensure remote /tmp/config.json exists + raw: "test -s {{ remote_cfg }}" + register: remote_cfg_check + changed_when: false + + - name: Stop if /tmp/config.json missing (soft-fail) + when: remote_cfg_check.rc != 0 + block: + - name: Journal soft-fail (missing config.json) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail — missing /tmp/config.json' + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + - name: Check controller tools (jq/sshpass/diff/sha256sum) + delegate_to: localhost + shell: | + command -v jq >/dev/null 2>&1 \ + && command -v sshpass >/dev/null 2>&1 \ + && command -v diff >/dev/null 2>&1 \ + && (command -v sha256sum >/dev/null 2>&1 || command -v busybox >/dev/null 2>&1) + args: { executable: /bin/bash } + register: ctrl_tools + changed_when: false + + - name: Stop if controller tools missing (soft-fail) + when: ctrl_tools.rc != 0 + block: + - name: Journal soft-fail (missing tools) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail — controller missing jq/sshpass/diff/sha256sum' + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + ########################################################################## + # b) Due diligence (external script, run on controller) + ########################################################################## + - name: Copy due diligence script to controller tmp + delegate_to: localhost + copy: + src: "{{ dd_script_src }}" + dest: "{{ dd_script }}" + mode: "0755" + + - name: Pull /tmp/config.json from device to controller + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:{{ remote_cfg }} + {{ local_cfg }} + + - name: Run due diligence on controller + delegate_to: localhost + shell: "sh {{ dd_script | quote }} {{ local_cfg | quote }} > {{ local_dd | quote }}" + args: { executable: /bin/sh } + changed_when: false + + - name: Parse due diligence JSON + delegate_to: localhost + set_fact: + dd: "{{ lookup('file', local_dd) | from_json }}" + + # Normalize types for robust `when:` checks + - name: Extract & normalize dd facts + delegate_to: localhost + set_fact: + w0_total_i: "{{ (dd.radios.wifi0.vaps_total | int) }}" + w1_total_i: "{{ (dd.radios.wifi1.vaps_total | int) }}" + w0_idx_i: "{{ (dd.radios.wifi0.indices | map('int') | list) }}" + w1_idx_i: "{{ (dd.radios.wifi1.indices | map('int') | list) }}" + + ########################################################################## + # c) Soft-fail: layout policy checks (one journal) + ########################################################################## + - name: Init soft-fail reasons (policy) + delegate_to: localhost + set_fact: + softfail_reasons: [] + + - name: wifi0 — add reason if ONE_VAP but index not 0/1 + delegate_to: localhost + when: (w0_total_i | int) == 1 and (0 not in w0_idx_i and 1 not in w0_idx_i) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi0 ONE_VAP but index not 0/1 (indices=' ~ (w0_idx_i | string) ~ ')' ] }}" + + - name: wifi0 — add reason if TWO_VAPS but indices not (0,1)/(1,0) + delegate_to: localhost + when: (w0_total_i | int) == 2 and (w0_idx_i | sort not in [[0,1],[1,0]]) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi0 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w0_idx_i | string) ~ ')' ] }}" + + - name: wifi0 — add reason if MORE_THAN_TWO_VAPS + delegate_to: localhost + when: (w0_total_i | int) > 2 + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi0 MORE_THAN_TWO_VAPS (indices=' ~ (w0_idx_i | string) ~ ')' ] }}" + + - name: wifi1 — add reason if ONE_VAP but index not 0/1 + delegate_to: localhost + when: (w1_total_i | int) == 1 and (0 not in w1_idx_i and 1 not in w1_idx_i) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi1 ONE_VAP but index not 0/1 (indices=' ~ (w1_idx_i | string) ~ ')' ] }}" + + - name: wifi1 — add reason if TWO_VAPS but indices not (0,1)/(1,0) + delegate_to: localhost + when: (w1_total_i | int) == 2 and (w1_idx_i | sort not in [[0,1],[1,0]]) + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi1 TWO_VAPS but indices not (0,1)/(1,0) (indices=' ~ (w1_idx_i | string) ~ ')' ] }}" + + - name: wifi1 — add reason if MORE_THAN_TWO_VAPS + delegate_to: localhost + when: (w1_total_i | int) > 2 + set_fact: + softfail_reasons: "{{ softfail_reasons + [ 'wifi1 MORE_THAN_TWO_VAPS (indices=' ~ (w1_idx_i | string) ~ ')' ] }}" + + - name: Publish soft-fail journal & stop host (policy) + when: (softfail_reasons | length) > 0 + block: + - name: Journal soft-fail (policy) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail (policy) — ' ~ (softfail_reasons | join('; ')) + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + ########################################################################## + # d) Filters to controller tmp + candidate generation (per radio) + ########################################################################## + - name: Copy jq filters to controller tmp + delegate_to: localhost + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "0644" + loop: + - { src: "{{ f_1vap0_src }}", dest: "{{ f_1vap0 }}" } + - { src: "{{ f_1vap1_src }}", dest: "{{ f_1vap1 }}" } + - { src: "{{ f_2_01_src }}", dest: "{{ f_2_01 }}" } + - { src: "{{ f_2_10_src }}", dest: "{{ f_2_10 }}" } + + - name: Start from current config as working file + delegate_to: localhost + copy: + src: "{{ local_cfg }}" + dest: "{{ local_step }}" + mode: "0644" + + # wifi0 choice + - name: wifi0 — apply filter 1vap_id0 + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 1 and (0 in w0_idx_i) + changed_when: true + + - name: wifi0 — apply filter 1vap_id1 (Policy B) + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 1 and (1 in w0_idx_i) + changed_when: true + + - name: wifi0 — apply filter 2vaps_0_1 + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 2 and (w0_idx_i | sort == [0,1]) + changed_when: true + + - name: wifi0 — apply filter 2vaps_1_0 (actual order 1,0) + delegate_to: localhost + shell: "jq --arg r wifi0 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w0_total_i | int) == 2 and (w0_idx_i == [1,0]) + changed_when: true + + # wifi1 choice + - name: wifi1 — apply filter 1vap_id0 + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_1vap0 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 1 and (0 in w1_idx_i) + changed_when: true + + - name: wifi1 — apply filter 1vap_id1 (Policy B) + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_1vap1 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 1 and (1 in w1_idx_i) + changed_when: true + + - name: wifi1 — apply filter 2vaps_0_1 + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_2_01 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 2 and (w1_idx_i | sort == [0,1]) + changed_when: true + + - name: wifi1 — apply filter 2vaps_1_0 (actual order 1,0) + delegate_to: localhost + shell: "jq --arg r wifi1 -f {{ f_2_10 | quote }} {{ local_step | quote }} > {{ local_step | quote }}.next && mv {{ local_step | quote }}.next {{ local_step | quote }}" + args: { executable: /bin/bash } + when: (w1_total_i | int) == 2 and (w1_idx_i == [1,0]) + changed_when: true + + - name: Debug normalized types + delegate_to: localhost + debug: + msg: + - "w0_total_i(type)={{ w0_total_i | type_debug }} value={{ w0_total_i }}" + - "w1_total_i(type)={{ w1_total_i | type_debug }} value={{ w1_total_i }}" + + - name: Move working file to final candidate + delegate_to: localhost + copy: + src: "{{ local_step }}" + dest: "{{ local_new }}" + mode: "0644" + + ########################################################################## + # e) Candidate checks — accumulate reasons; soft-fail once if any + ########################################################################## + - name: Init soft-fail reasons (candidate) + delegate_to: localhost + set_fact: + cand_reasons: [] + + - name: Check JSON validity + delegate_to: localhost + shell: "jq -e '.' {{ local_new | quote }} >/dev/null" + args: { executable: /bin/bash } + register: json_valid + changed_when: false + failed_when: false + + - name: Add reason if JSON invalid + delegate_to: localhost + when: json_valid.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'invalid JSON syntax in candidate' ] }}" + + - name: Semantic — wifi0 exactly one enabled AP VAP + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w0 + changed_when: false + failed_when: false + + - name: Add reason if wifi0 semantic fails + delegate_to: localhost + when: sem_w0.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi0 must have exactly one enabled AP VAP' ] }}" + + - name: Semantic — wifi1 exactly one enabled AP VAP + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w1 + changed_when: false + failed_when: false + + - name: Add reason if wifi1 semantic fails + delegate_to: localhost + when: sem_w1.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi1 must have exactly one enabled AP VAP' ] }}" + + - name: Semantic — wifi0 enabled AP VAP has lbd=true + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w0_lbd + changed_when: false + failed_when: false + + - name: Add reason if wifi0 lbd check fails + delegate_to: localhost + when: sem_w0_lbd.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi0 enabled AP VAP must have lbd=true' ] }}" + + - name: Semantic — wifi1 enabled AP VAP has lbd=true + delegate_to: localhost + shell: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ local_new | quote }} >/dev/null + args: { executable: /bin/bash } + register: sem_w1_lbd + changed_when: false + failed_when: false + + - name: Add reason if wifi1 lbd check fails + delegate_to: localhost + when: sem_w1_lbd.rc != 0 + set_fact: + cand_reasons: "{{ cand_reasons + [ 'wifi1 enabled AP VAP must have lbd=true' ] }}" + + - name: Get original file size + delegate_to: localhost + stat: + path: "{{ local_cfg }}" + register: stat_old + changed_when: false + + - name: Get new file size + delegate_to: localhost + stat: + path: "{{ local_new }}" + register: stat_new + changed_when: false + + - name: Add reason if size delta > 2% + delegate_to: localhost + when: stat_old.stat.size | int == 0 or + ( ((stat_new.stat.size | int) - (stat_old.stat.size | int)) | abs ) > ( (stat_old.stat.size | int) * 0.02 ) + set_fact: + cand_reasons: "{{ cand_reasons + [ 'candidate size delta >2% (old=' ~ (stat_old.stat.size | string) ~ ', new=' ~ (stat_new.stat.size | string) ~ ')' ] }}" + + - name: Build unified diff (first 200 lines) + delegate_to: localhost + shell: "diff -u {{ local_cfg | quote }} {{ local_new | quote }} | sed -n '1,200p' > {{ local_diff | quote }} || true" + args: { executable: /bin/bash } + changed_when: false + + - name: Show unified diff (first 200 lines) + delegate_to: localhost + debug: + msg: "{{ lookup('file', local_diff) | default('(no diff output)') }}" + + - name: sha256 (controller) of candidate + delegate_to: localhost + command: sha256sum {{ local_new }} + register: sha_local + changed_when: false + + - name: Copy candidate to device temp + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ local_new }} + {{ ssh_user }}@{{ ansible_host }}:{{ remote_new }} + register: scp_new + changed_when: true + + - name: sha256 (remote) of candidate + raw: "sha256sum {{ remote_new }} || busybox sha256sum {{ remote_new }}" + register: sha_remote + changed_when: false + + - name: Add reason if sha256 mismatch + delegate_to: localhost + when: (sha_local.stdout.split()[0]) != (sha_remote.stdout.split()[0]) + set_fact: + cand_reasons: "{{ cand_reasons + [ 'sha256 mismatch controller vs remote' ] }}" + + - name: Detect SSIDs with NBSP (informational) + delegate_to: localhost + shell: | + nbsp="$(printf '\302\240')" + jq -r '..|objects|.ssid? // empty' {{ local_new | quote }} | grep -F "$nbsp" || true + args: { executable: /bin/bash } + register: nbsp_lines + changed_when: false + + - name: Save NBSP report lines + set_fact: + ssid_nbsp_lines: "{{ nbsp_lines.stdout_lines | default([]) }}" + + - name: Publish soft-fail journal & stop host (candidate issues) + when: (cand_reasons | length) > 0 + block: + - name: Journal soft-fail (candidate) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': 'multissidfix: soft-fail (candidate) — ' ~ (cand_reasons | join('; ')) + } | to_json }}" + payload_encoding: "string" + changed_when: false + - meta: end_host + + ########################################################################## + # f) Backup, parachute, promote, persist, apply, outcome + ########################################################################## + - name: Backup current config on device + raw: "cp -a {{ remote_cfg }} {{ remote_backup }}" + changed_when: true + + # --- PRE-APPLY HEADS-UP --- + - name: Journal — candidate validated, will promote/persist/apply shortly + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'multissidfix: candidate passed checks; proceeding to backup + parachute + promote + sysconf -w + system-stop/start. ' + ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Arm reboot parachute (BusyBox reboot -d {{ parachute_delay_s }}) + raw: "sh -c 'trap \"\" HUP; reboot -d {{ parachute_delay_s }} /dev/null 2>&1 &'" + changed_when: true + ignore_errors: true + + - name: Promote candidate to active config + raw: "mv {{ remote_new }} {{ remote_cfg }} && chown root:root {{ remote_cfg }} && chmod 0644 {{ remote_cfg }}" + changed_when: true + + # IMPORTANT: persist BEFORE runtime restart + - name: Persist config to flash (sysconf -w) + raw: "sysconf -w" + register: sysconf_write + changed_when: true + failed_when: false + + # --- PRE-RESTART COMMIT --- + - name: Pre-restart journal (promoted & persisted; about to restart from controller) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'multissidfix: promoted candidate, persisted (sysconf -w rc=' ~ (sysconf_write.rc | default('n/a')) | string ~ '), ' + ~ 'system-stop/start launching from controller with 10s cap; ' + ~ 'parachute -d ' ~ parachute_delay_s|string ~ 's armed. ' + ~ 'size(old/new)=' ~ stat_old.stat.size|string ~ '/' ~ stat_new.stat.size|string + ~ ', nbsp_ssids=' ~ (ssid_nbsp_lines|length)|string + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Launch restart from controller with 10s cap (SSH command) + delegate_to: localhost + shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'" + args: + executable: /bin/bash + register: restart_kick + changed_when: true + failed_when: false + + - name: Note restart_kick result (debug) + delegate_to: localhost + debug: + msg: + - "restart_kick.rc={{ restart_kick.rc }}" + - "stdout(last 5 lines): {{ (restart_kick.stdout_lines | default([]))[-5:] | default([]) }}" + - "stderr(last 5 lines): {{ (restart_kick.stderr_lines | default([]))[-5:] | default([]) }}" + + # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3) + - name: Probe SSH with nc (24 tries, 5s each) + delegate_to: localhost + shell: "nc -z -w 3 {{ ansible_host }} 22" + register: nc_probe + retries: 24 + delay: 5 + until: nc_probe.rc == 0 + changed_when: false + failed_when: false + + - name: Set ssh_up fact + delegate_to: localhost + set_fact: + ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}" + + # --- SUCCESS PATH --- + - name: Post-return actions (only if SSH came back) + when: ssh_up | bool + block: + + - name: Disarm delayed reboot if present + raw: "pgrep -x reboot && kill -9 $(pgrep -x reboot) || true" + changed_when: true + ignore_errors: true + register: disarm_reboot + + # Re-run simple semantic checks on the active config + - name: Device semantic — wifi0 exactly one enabled AP VAP + raw: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w0 + changed_when: false + failed_when: false + + - name: Device semantic — wifi1 exactly one enabled AP VAP + raw: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w1 + changed_when: false + failed_when: false + + - name: Device semantic — wifi0 enabled AP VAP has lbd=true + raw: | + jq -e '(.wireless.radios.wifi0.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w0_lbd + changed_when: false + failed_when: false + + - name: Device semantic — wifi1 enabled AP VAP has lbd=true + raw: | + jq -e '(.wireless.radios.wifi1.vaps // []) + | map(select(.mode=="ap" and (.enabled==true) and (.lbd==true))) | length == 1' {{ remote_cfg }} >/dev/null + register: dev_sem_w1_lbd + changed_when: false + failed_when: false + + - name: sha256 (remote) of current active config (post-return) + raw: "sha256sum {{ remote_cfg }} || busybox sha256sum {{ remote_cfg }}" + register: sha_remote_after + changed_when: false + failed_when: false + + - name: Decide post-return checks summary (controller side) + delegate_to: localhost + set_fact: + multissid_post_summary: + reboot_disarmed: "{{ (disarm_reboot is defined) and (disarm_reboot.rc is defined) and (disarm_reboot.rc in [0]) }}" + persisted_ok: "{{ (sysconf_write.rc | default(0)) == 0 }}" + dev_sem_ok: + w0_one: "{{ (dev_sem_w0.rc | default(1)) == 0 }}" + w1_one: "{{ (dev_sem_w1.rc | default(1)) == 0 }}" + w0_lbd: "{{ (dev_sem_w0_lbd.rc | default(1)) == 0 }}" + w1_lbd: "{{ (dev_sem_w1_lbd.rc | default(1)) == 0 }}" + cfg_hash_match: "{{ (sha_remote_after.stdout.split()[0] | default('')) == (sha_remote.stdout.split()[0] | default('')) }}" + old_size: "{{ stat_old.stat.size | default('n/a') }}" + new_size: "{{ stat_new.stat.size | default('n/a') }}" + nbsp_count: "{{ (ssid_nbsp_lines | default([])) | length }}" + + - name: Publish final success journal (device returned; persisted; checks pass) + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': + ( + 'multissidfix: success — device is back; ' + ~ (multissid_post_summary.reboot_disarmed | ternary('parachute disarmed; ', 'parachute was not armed; ')) + ~ (multissid_post_summary.persisted_ok | ternary('persisted (sysconf -w); ', 'persist failed; ')) + ~ 'semantics: ' + ~ 'w0_one=' ~ (multissid_post_summary.dev_sem_ok.w0_one | string) ~ ', ' + ~ 'w1_one=' ~ (multissid_post_summary.dev_sem_ok.w1_one | string) ~ ', ' + ~ 'w0_lbd=' ~ (multissid_post_summary.dev_sem_ok.w0_lbd | string) ~ ', ' + ~ 'w1_lbd=' ~ (multissid_post_summary.dev_sem_ok.w1_lbd | string) ~ '; ' + ~ 'cfg_match=' ~ (multissid_post_summary.cfg_hash_match | string) ~ '; ' + ~ 'size(old/new)=' ~ (multissid_post_summary.old_size | string) ~ '/' ~ (multissid_post_summary.new_size | string) ~ '; ' + ~ 'nbsp_ssids=' ~ (multissid_post_summary.nbsp_count | string) + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Set NetBox custom field multissidfix=v1 + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'multissidfix', + 'task_result': 'v1' + } | to_json }}" + payload_encoding: "string" + changed_when: false + + + # --- FAILURE PATH (device did not return) --- + - name: Journal — restart/SSH probe failed + when: not ssh_up + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': ( + 'multissidfix: restart issued; SSH did not return after 24 x 5s checks — leaving parachute active.' + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Stop host after failed SSH probe + when: not ssh_up + meta: end_host diff --git a/files/ansible-playbooks/persuasive-upgrade.yml b/files/ansible-playbooks/persuasive-upgrade.yml new file mode 100644 index 0000000..b287a63 --- /dev/null +++ b/files/ansible-playbooks/persuasive-upgrade.yml @@ -0,0 +1,330 @@ +--- +- name: Persuasive / hunting upgrade orchestrator + hosts: all + gather_facts: no + + vars: + # RabbitMQ config + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + + # Exchanges / queues + work_exchange: "deviceconfig" + work_routing_key: "deviceconfig" # immediate work path + holding_exchange: "deviceconfig.holding" + holding_routing_key: "persuasive" # reschedules go to persuasive holding + control_exchange: "controls" + control_queue: "queue_controls" + + # Probing defaults + tcp_port: 22 + nc_timeout: 5 + ssh_timeout: 10 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + + # Policy defaults (overridable via -e from task_options) + pu_period_default: "30m" + pu_attempts_default: 48 + pu_untilhours_default: "72h" + + tasks: + - name: Normalize inputs (no self-referential defaults) + ansible.builtin.set_fact: + attempt: "{{ (attempt | default(1)) | int }}" + pu_period: "{{ pu_period | default(pu_period_default) }}" + pu_attempts: "{{ (pu_attempts | default(pu_attempts_default)) | int }}" + pu_untilhours: "{{ pu_untilhours | default(pu_untilhours_default) }}" + correlation_id: "{{ correlation_id | default('') }}" + original_emitted_at: "{{ original_emitted_at | default(lookup('pipe','date -u +%FT%TZ')) }}" + controller_now_iso: "{{ lookup('pipe','date -u +%FT%TZ') }}" + + - name: Show received metadata + ansible.builtin.debug: + msg: + - "attempt={{ attempt }}" + - "pu_period={{ pu_period }}" + - "pu_attempts={{ pu_attempts }}" + - "pu_untilhours={{ pu_untilhours }}" + - "original_emitted_at={{ original_emitted_at }}" + - "controller_now={{ controller_now_iso }}" + + # Optional time budget + - name: Compute budget_ms (supports d/h/m/s in pu_untilhours) + ansible.builtin.set_fact: + budget_ms: >- + {{ + ( + ( + (pu_untilhours | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 + + (pu_untilhours | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 + + (pu_untilhours | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 + + (pu_untilhours | regex_findall('([0-9]+)s') | first | default('0')) | int + ) * 1000 + ) + }} + + - name: Compute elapsed since original emission (ms) + ansible.builtin.set_fact: + elapsed_ms: >- + {{ + ( + (lookup('pipe', 'date -u -d ' ~ controller_now_iso ~ ' +%s') | int) - + (lookup('pipe', 'date -u -d ' ~ original_emitted_at ~ ' +%s') | int) + ) * 1000 + }} + + # Helpers for final summary line + - name: Compute period_sec from pu_period (supports d/h/m/s) + ansible.builtin.set_fact: + period_sec: >- + {{ + ( + (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 + + (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 + + (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 + + (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int + ) + }} + + - name: Compute budget_hours string (one decimal) + ansible.builtin.set_fact: + budget_hours_str: "{{ '%.1f' | format( (budget_ms | int) / 3600000.0 ) }}" + + - name: Gave up due to time budget + when: (elapsed_ms | int) >= (budget_ms | int) + delegate_to: localhost + block: + - name: Journal final give-up (unified message) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ( + "device wasn't reachable for " ~ (attempt | string) ~ + " attempts, each " ~ (period_sec | string) ~ + " seconds for " ~ budget_hours_str ~ + " hours. backing off, won't persuade it more. Schedule again if needed (limit: time budget exhausted)" + ) + } | to_json + }} + payload_encoding: "string" + register: rmq_j_budget + changed_when: (rmq_j_budget.json is defined) and (rmq_j_budget.json.routed | default(false) | bool) + + - ansible.builtin.meta: end_host + + # Attempt cap + - name: Gave up due to attempts cap + when: (attempt | int) >= (pu_attempts | int) + delegate_to: localhost + block: + - name: Journal final give-up (unified message) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ( + "device wasn't reachable for " ~ (attempt | string) ~ + " attempts, each " ~ (period_sec | string) ~ + " seconds for " ~ budget_hours_str ~ + " hours. backing off, won't persuade it more. Schedule again if needed (limit: attempts cap reached)" + ) + } | to_json + }} + payload_encoding: "string" + register: rmq_j_gaveup + changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool) + + - ansible.builtin.meta: end_host + + # Probe + - name: Check TCP/{{ tcp_port }} via nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + # ONLINE → hand off to normal path + - name: Publish upgrade-confirmed to deviceconfig (immediate) + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ work_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ work_routing_key }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "upgrade-confirmed" + } | to_json + }} + payload_encoding: "string" + register: rmq_start + changed_when: (rmq_start.json is defined) and (rmq_start.json.routed | default(false) | bool) + + - name: "Journal: online, starting upgrade" + when: nc_probe.rc == 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ("persuasive-upgrade: device online, starting upgrade-confirmed now. Original=" ~ original_emitted_at) + } | to_json + }} + payload_encoding: "string" + register: rmq_j_start + changed_when: (rmq_j_start.json is defined) and (rmq_j_start.json.routed | default(false) | bool) + + - ansible.builtin.meta: end_host + when: nc_probe.rc == 0 + + # OFFLINE → reschedule into persuasive holding + - name: Compute TTL ms from pu_period (supports d/h/m/s) + when: nc_probe.rc != 0 + ansible.builtin.set_fact: + ttl_ms: >- + {{ + ( + ( + (pu_period | regex_findall('([0-9]+)d') | first | default('0')) | int * 24 * 60 * 60 + + (pu_period | regex_findall('([0-9]+)h') | first | default('0')) | int * 60 * 60 + + (pu_period | regex_findall('([0-9]+)m') | first | default('0')) | int * 60 + + (pu_period | regex_findall('([0-9]+)s') | first | default('0')) | int + ) * 1000 + ) + }} + + - name: Compute next_attempt + when: nc_probe.rc != 0 + ansible.builtin.set_fact: + next_attempt: "{{ attempt | int + 1 }}" + + - name: Build next task_options (carry policy + increment attempt) + when: nc_probe.rc != 0 + ansible.builtin.set_fact: + next_task_options: >- + -e pu_period={{ pu_period }} + -e pu_attempts={{ pu_attempts }} + -e pu_untilhours={{ pu_untilhours }} + -e attempt={{ next_attempt }} + -e original_emitted_at='{{ original_emitted_at }}' + + - name: Decide if we should emit the reschedule journal this attempt + when: nc_probe.rc != 0 + ansible.builtin.set_fact: + _pu_journal_this_try: "{{ (attempt | int) in [1, 2] or ((attempt | int) % 10 == 0) }}" + + - name: "Publish delayed persuasive-upgrade to holding (routing: persuasive)" + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ ttl_ms | int }}" + routing_key: "{{ holding_routing_key }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "persuasive-upgrade", + "task_options": (next_task_options | trim) + } | to_json + }} + payload_encoding: "string" + register: rmq_pub_next + changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool) + + - name: "Journal: offline, rescheduled" + when: nc_probe.rc != 0 and (_pu_journal_this_try | bool) + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ control_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ("persuasive-upgrade: device offline, rescheduling (attempt " ~ attempt ~ "/" ~ pu_attempts ~ ", next in " ~ pu_period ~ "). Original=" ~ original_emitted_at) + } | to_json + }} + payload_encoding: "string" + register: rmq_j_off + changed_when: (rmq_j_off.json is defined) and (rmq_j_off.json.routed | default(false) | bool) + + - ansible.builtin.meta: end_host + when: nc_probe.rc != 0 diff --git a/files/ansible-playbooks/remove-scroll24.yml b/files/ansible-playbooks/remove-scroll24.yml new file mode 100644 index 0000000..ab5e7e7 --- /dev/null +++ b/files/ansible-playbooks/remove-scroll24.yml @@ -0,0 +1,158 @@ +--- +- name: Remove scroll24 script and cron references + hosts: all + gather_facts: no + + vars: + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + remote_script: "/root/scroll24.sh" + remote_crontab: "/etc/crontabs/root" + tmp_cron_new: "/tmp/cron.root.new" + crontab_backup_dir: "/etc/crontabs" + + # RabbitMQ (same contract you use elsewhere) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + tasks: + ########################################################################## + # 1) Make sure crontab file exists (don’t create noise) + ########################################################################## + - name: Ensure crontab file exists (with perms) + raw: | + if [ ! -f {{ remote_crontab }} ]; then + touch {{ remote_crontab }}; + fi + chown root:root {{ remote_crontab }}; + chmod 0644 {{ remote_crontab }}; + changed_when: false + + ########################################################################## + # 2) Check whether any scroll24 entries exist + ########################################################################## + - name: Detect existing scroll24 lines + raw: "grep -F 'scroll24.sh' {{ remote_crontab }} || true" + register: cron_subset + changed_when: false + failed_when: false + + - name: Decide if crontab needs cleanup + set_fact: + crontab_changed: "{{ (cron_subset.stdout | trim) != '' }}" + + - name: Backup current crontab (timestamped) + when: crontab_changed | bool + raw: "cp -a {{ remote_crontab }} {{ crontab_backup_dir }}/root.bak.$(date +%Y%m%d%H%M%S)" + changed_when: true + + - name: Remove scroll24 lines from crontab (preserve others; tidy EOF) + when: crontab_changed | bool + raw: "grep -v 'scroll24\\.sh' {{ remote_crontab }} > {{ tmp_cron_new }} && awk 'BEGIN{for(i=1;i<=NR;i++)a[i]=$0} {a[NR]=$0} END{e=NR; while(e>0 && a[e] ~ /^[[:space:]]*$/){e--}; for(i=1;i<=e;i++) print a[i]}' {{ tmp_cron_new }} > {{ tmp_cron_new }}.trim && mv {{ tmp_cron_new }}.trim {{ tmp_cron_new }} && printf '\\n' >> {{ tmp_cron_new }} && mv {{ tmp_cron_new }} {{ remote_crontab }} && chown root:root {{ remote_crontab }} && chmod 0644 {{ remote_crontab }}" + changed_when: true + + ########################################################################## + # 3) Remove /root/scroll24.sh if present + ########################################################################## + - name: Check if /root/scroll24.sh exists + raw: "[ -f {{ remote_script }} ] && echo PRESENT || echo ABSENT" + register: script_check + changed_when: false + + - name: Remove /root/scroll24.sh + when: (script_check.stdout | trim) == 'PRESENT' + raw: "rm -f {{ remote_script }}" + register: rm_script + changed_when: true + + - name: Flag script_removed + set_fact: + script_removed: "{{ ((script_check.stdout | trim) == 'PRESENT') }}" + + ########################################################################## + # 4) If crontab changed, restart crond (with :51–:59 guard) + ########################################################################## + - name: Get current seconds + when: crontab_changed | bool + raw: "date +%S" + register: nowsec + changed_when: false + + - name: Sleep 10s if seconds 51-59 + when: crontab_changed | bool and (nowsec.stdout | int >= 51) + pause: + seconds: 10 + + - name: Restart crond via move/move + when: crontab_changed | bool + raw: "mv /tmp/launchd/services/crond /root/crond && sleep 1 && mv /root/crond /tmp/launchd/services/crond" + register: crond_restart + changed_when: true + failed_when: false + + - name: Verify crond is running + when: crontab_changed | bool + raw: "pgrep -f '/usr/sbin/crond' || busybox pgrep crond || echo missing" + register: crond_pid + changed_when: false + failed_when: false + + ########################################################################## + # 5) Journal + reset custom field + ########################################################################## + - name: Publish removal journal to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'journal_add', + 'task_result': + ( + 'scroll24: removal — ' + ~ (crontab_changed | ternary('crontab cleaned; ', 'no crontab refs; ')) + ~ (script_removed | ternary('script deleted; ', 'script not present; ')) + ~ (crontab_changed | ternary('crond restarted', 'crond unchanged')) + ) + } | to_json }}" + payload_encoding: "string" + changed_when: false + + - name: Clear NetBox custom field scroll24 + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: { content-type: "application/json" } + body_format: json + body: + properties: { content_type: "application/json" } + routing_key: "{{ control_queue }}" + payload: "{{ { + 'inscope_device': (ansible_hostname | default(inventory_hostname)), + 'task_name': 'custom_field_set', + 'task_add1': 'scroll24', + 'task_result': 'nomore' + } | to_json }}" + payload_encoding: "string" + changed_when: false diff --git a/files/ansible-playbooks/resetradios.yml b/files/ansible-playbooks/resetradios.yml new file mode 100644 index 0000000..a75e0ca --- /dev/null +++ b/files/ansible-playbooks/resetradios.yml @@ -0,0 +1,33 @@ +--- +- name: Reset radios on a host and show brief status + hosts: all + gather_facts: no + + tasks: + - name: Bounce ath0 and ath1 with ifconfig + ansible.builtin.raw: | + set -e + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + ifconfig ath0 down + sleep 2 + ifconfig ath0 up + ifconfig ath1 down + sleep 1 + ifconfig ath1 up + register: reset_out + changed_when: true + + - name: Wait 2s for interfaces to settle (controller-side) + ansible.builtin.pause: + seconds: 2 + + - name: Grab brief ath0/ath1 status (header + one following line) + ansible.builtin.raw: | + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + iwconfig 2>/dev/null | grep -E '^(ath0|ath1)\b' -A 1 + register: iw_out + changed_when: false + + - name: Show ath0/ath1 status + ansible.builtin.debug: + msg: "{{ iw_out.stdout | trim }}" diff --git a/files/ansible-playbooks/setntptime.yml b/files/ansible-playbooks/setntptime.yml new file mode 100644 index 0000000..ea91ac5 --- /dev/null +++ b/files/ansible-playbooks/setntptime.yml @@ -0,0 +1,39 @@ +--- +- name: Simple script to show current version + hosts: all + gather_facts: no + + tasks: + - name: get the date + ansible.builtin.raw: | + set -e + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + date + register: date_out + changed_when: true + + + - name: show the current date + ansible.builtin.debug: + msg: "{{ date_out.stdout | trim }}" + + - name: use sntp -S to set system time + ansible.builtin.raw: | + set -e + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + sntp -S time.ikeja.co.za + changed_when: true + + - name: get the date + ansible.builtin.raw: | + set -e + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + date + register: date_out + changed_when: true + + + - name: show the current date + ansible.builtin.debug: + msg: "{{ date_out.stdout | trim }}" + diff --git a/files/ansible-playbooks/system-check-version.yml b/files/ansible-playbooks/system-check-version.yml new file mode 100644 index 0000000..af621e1 --- /dev/null +++ b/files/ansible-playbooks/system-check-version.yml @@ -0,0 +1,152 @@ +--- +- name: System firmware version check (banner probe + journal) + hosts: all + gather_facts: no + + # RabbitMQ (match your existing defaults) + vars: + rmq_host: "10.210.12.2" + rmq_port: 15672 + rmq_user: "admin" + rmq_pass: "change_me" + rmq_vhost: "app" + rmq_exchange: "controls" + control_queue: "queue_controls" + + # Probing/SSH defaults (kept identical to your afterupgrade_check.yml) + tcp_port: 22 + nc_timeout: 5 + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}" + ssh_timeout: 10 + + tasks: + # -------- Fast TCP reachability probe (controller-side), unchanged style -------- + - name: Check if TCP/{{ tcp_port }} is reachable with nc + delegate_to: localhost + ansible.builtin.shell: | + nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }} + register: nc_probe + changed_when: false + ignore_errors: true + + # If TCP failed → emit a single journal line and stop + - name: Build TCP-fail journal payload + when: nc_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + tcp_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + system check: TCP {{ tcp_port }} unreachable (nc failed) + + - name: Publish TCP-fail journal + when: tcp_fail_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tcp_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_tcp_fail_resp + changed_when: (rmq_tcp_fail_resp.json is defined) and (rmq_tcp_fail_resp.json.routed | default(false) | bool) + + - name: Stop host after TCP failure + when: nc_probe.rc != 0 + ansible.builtin.meta: end_host + + # -------- SSH banner probe (controller-side), EXACT command reused -------- + - name: Probe banner via SSH from controller (classic extraction) + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ ssh_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ ssh_pass }}' \ + ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \ + "${USER}@${HOST}" \ + "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1" + register: banner_probe + changed_when: false + ignore_errors: true + + # SSH error → journal and stop + - name: Build SSH-fail journal payload + when: banner_probe.rc != 0 + delegate_to: localhost + ansible.builtin.set_fact: + ssh_fail_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + system check: SSH error - {{ (banner_probe.stderr | default('') | trim) }} + + - name: Publish SSH-fail journal + when: ssh_fail_payload is defined + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ ssh_fail_payload | to_json }}" + payload_encoding: "string" + register: rmq_ssh_fail_resp + changed_when: (rmq_ssh_fail_resp.json is defined) and (rmq_ssh_fail_resp.json.routed | default(false) | bool) + + - name: Stop host after SSH failure + when: banner_probe.rc != 0 + ansible.builtin.meta: end_host + + # -------- Success: banner line captured → journal with "system check:" prefix -------- + - name: Build success journal payload (banner captured) + delegate_to: localhost + ansible.builtin.set_fact: + sc_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + system check: Banner='{{ (banner_probe.stdout | default('') | trim) }}' + + - name: Publish success journal + delegate_to: localhost + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ sc_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_sc_success_resp + changed_when: (rmq_sc_success_resp.json is defined) and (rmq_sc_success_resp.json.routed | default(false) | bool) + diff --git a/files/ansible-playbooks/system-stop-system-start.yml b/files/ansible-playbooks/system-stop-system-start.yml new file mode 100644 index 0000000..4e5f8dd --- /dev/null +++ b/files/ansible-playbooks/system-stop-system-start.yml @@ -0,0 +1,19 @@ +--- +- name: Simple script restart all processes + hosts: all + gather_facts: no + + tasks: + - name: execute and grab output + ansible.builtin.raw: | + set -e + PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH + system-stop ; sleep 1; system-start + register: ssss_out + changed_when: true + + - name: show the output + ansible.builtin.debug: + msg: "{{ ssss_out.stdout | trim }}" + + diff --git a/files/ansible-playbooks/update-indoor.yml b/files/ansible-playbooks/update-indoor.yml new file mode 100644 index 0000000..7ce044a --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml @@ -0,0 +1,1221 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/HOURS)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_hours: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # Compute effective seconds from HOURS (0h → ~20 seconds grace); clamp non-negative + - name: "Normalize rebootin (phase 3: compute seconds from HOURS)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_seconds: >- + {{ + 20 + if (_reboot_requested | default(false)) and ((_reboot_hours | int) <= 0) + else + (((_reboot_hours | int) * 3600)) + if (_reboot_requested | default(false)) + else 0 + }} + + # Derive minutes from seconds (non-negative), for tasks/summary that expect minutes + - name: "Normalize rebootin (phase 4: derive minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_minutes: "{{ ((_reboot_seconds | int) // 60) if (_reboot_seconds | int) > 0 else 0 }}" + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build control queue payload for indoor-restart-scheduled tag + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + + # ──────────────────────────────── NEW: derive expected version for the checker ──────────────────────────────── + + # --- SAFE expected target derivation (split into separate set_fact tasks) --- + - name: Derive expected indoor firmware from image filename (safe – base) + delegate_to: localhost + ansible.builtin.set_fact: + _img_base: "{{ image_filename | regex_replace('\\.bin$','') }}" + when: au_delay_sec is defined + + - name: Derive X.Y.Z core from image filename (if any) + delegate_to: localhost + ansible.builtin.set_fact: + _img_ver_core: >- + {{ + (_img_base is search('[0-9]+\\.[0-9]+\\.[0-9]+')) + | ternary( + (_img_base | regex_replace('.*?([0-9]+\\.[0-9]+\\.[0-9]+).*','\\1')), + '' + ) + }} + when: au_delay_sec is defined + + - name: Derive rev number from -rNNNN (preferred) + delegate_to: localhost + ansible.builtin.set_fact: + _img_rev_num: >- + {{ + (_img_base is search('-r[0-9]+')) + | ternary( + (_img_base | regex_replace('.*-r([0-9]+).*','\\1')), + '' + ) + }} + when: au_delay_sec is defined + + - name: If no -rNNNN, try 'rev NNNN' (case-insensitive) + delegate_to: localhost + ansible.builtin.set_fact: + _img_rev_num: >- + {{ + (_img_rev_num | length) > 0 + | ternary( + _img_rev_num, + ( + (_img_base is search('(?i)rev[^0-9]*[0-9]+')) + | ternary( + (_img_base | regex_replace('.*(?i)rev[^0-9]*([0-9]+).*','\\1')), + '' + ) + ) + ) + }} + when: au_delay_sec is defined + + - name: Assemble expected_fw_core (prefer X.Y.Z-rNNNN → X.Y.Z → image base) + delegate_to: localhost + ansible.builtin.set_fact: + expected_fw_core: >- + {{ + (_img_ver_core | length) > 0 and (_img_rev_num | length) > 0 + and (_img_ver_core ~ '-r' ~ _img_rev_num) + or + ((_img_ver_core | length) > 0 and _img_ver_core) + or + _img_base + }} + when: au_delay_sec is defined + + - name: Debug derived target version + delegate_to: localhost + ansible.builtin.debug: + msg: + - "image_filename={{ image_filename }}" + - "expected_fw_core={{ expected_fw_core }}" + when: expected_fw_core is defined + + + - name: If no -rNNNN, also try to pull rev from 'rev NNNN' (case-insensitive) + delegate_to: localhost + ansible.builtin.set_fact: + _img_rev_num: >- + {{ + (_img_rev_num | length) > 0 + | ternary( + _img_rev_num, + ( + (_img_base is search('(?i)rev[^0-9]*[0-9]+')) + | ternary( + (_img_base | regex_replace('.*(?i)rev[^0-9]*([0-9]+).*','\\1')), + '' + ) + ) + ) + }} + when: au_delay_sec is defined + + - name: Assemble expected_fw_core (prefer X.Y.Z-rNNNN → X.Y.Z → image base) + delegate_to: localhost + ansible.builtin.set_fact: + expected_fw_core: >- + {{ + (_img_ver_core | length) > 0 and (_img_rev_num | length) > 0 + and (_img_ver_core ~ '-r' ~ _img_rev_num) + or + ((_img_ver_core | length) > 0 and _img_ver_core) + or + _img_base + }} + when: au_delay_sec is defined + + - name: Debug derived target version + delegate_to: localhost + ansible.builtin.debug: + msg: + - "image_filename={{ image_filename }}" + - "expected_fw_core={{ expected_fw_core }}" + when: expected_fw_core is defined + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ expected_fw_core | default(fw_banner_repr | default(image_filename | regex_replace('\\.bin$',''))) }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-bckp1 b/files/ansible-playbooks/update-indoor.yml-bckp1 new file mode 100644 index 0000000..0574ccc --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml-bckp1 @@ -0,0 +1,997 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_minutes: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="$(( {{ _reboot_minutes | int }} * 60 ))" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + 'now' if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-beforefixrebootin b/files/ansible-playbooks/update-indoor.yml-beforefixrebootin new file mode 100644 index 0000000..96643cb --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml-beforefixrebootin @@ -0,0 +1,1108 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_minutes: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # >>> NEW: compute effective seconds (0 min → 20 seconds) + - name: "Normalize rebootin (phase 3: compute seconds)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_seconds: >- + {{ + 20 if (_reboot_requested | default(false)) and ((_reboot_minutes | int) == 0) + else ((_reboot_minutes | int) * 60) + }} + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build control queue payload for indoor-restart-scheduled tag + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + + - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix b/files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix new file mode 100644 index 0000000..1b93082 --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml-bfr_reboot_fix @@ -0,0 +1,1100 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_minutes: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'now' if _reboot_is_now else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="$(( {{ _reboot_minutes | int }} * 60 ))" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build control queue payload for indoor-restart-scheduled tag + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + 'now' if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + + - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ ((_reboot_minutes | int) * 60) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ ((_reboot_minutes | int) * 60) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-day2 b/files/ansible-playbooks/update-indoor.yml-day2 new file mode 100644 index 0000000..3d81c6e --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml-day2 @@ -0,0 +1,1117 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/HOURS)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_hours: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # Compute effective seconds from HOURS (0h → ~20 seconds grace); clamp non-negative + - name: "Normalize rebootin (phase 3: compute seconds from HOURS)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_seconds: >- + {{ + 20 + if (_reboot_requested | default(false)) and ((_reboot_hours | int) <= 0) + else + (((_reboot_hours | int) * 3600)) + if (_reboot_requested | default(false)) + else 0 + }} + + # Derive minutes from seconds (non-negative), for tasks/summary that expect minutes + - name: "Normalize rebootin (phase 4: derive minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_minutes: "{{ ((_reboot_seconds | int) // 60) if (_reboot_seconds | int) > 0 else 0 }}" + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build control queue payload for indoor-restart-scheduled tag + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + + - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml-fixingerrors b/files/ansible-playbooks/update-indoor.yml-fixingerrors new file mode 100644 index 0000000..3d81c6e --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml-fixingerrors @@ -0,0 +1,1117 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in HOURS (integer). Consumer always passes hours; 0 means immediate (~20s grace). + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early (HOURS) ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/HOURS)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_hours: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # Compute effective seconds from HOURS (0h → ~20 seconds grace); clamp non-negative + - name: "Normalize rebootin (phase 3: compute seconds from HOURS)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_seconds: >- + {{ + 20 + if (_reboot_requested | default(false)) and ((_reboot_hours | int) <= 0) + else + (((_reboot_hours | int) * 3600)) + if (_reboot_requested | default(false)) + else 0 + }} + + # Derive minutes from seconds (non-negative), for tasks/summary that expect minutes + - name: "Normalize rebootin (phase 4: derive minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_minutes: "{{ ((_reboot_seconds | int) // 60) if (_reboot_seconds | int) > 0 else 0 }}" + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build control queue payload for indoor-restart-scheduled tag + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + + - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-indoor.yml2 b/files/ansible-playbooks/update-indoor.yml2 new file mode 100644 index 0000000..96643cb --- /dev/null +++ b/files/ansible-playbooks/update-indoor.yml2 @@ -0,0 +1,1108 @@ +--- +# update-secondline.yml +- name: Second-line upgrade tunneled noninvasive engine + hosts: all + gather_facts: no + + vars: + # Busybox-safe PATH prefix for all remote raw calls on DEV1 + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + # DEV1 credentials (stable, like rebootin222) + dev1_user: "root" + dev1_pass: "wavewave" + + # Tunnel target DEV2 behind DEV1 + dev2_host: "192.168.1.1" + dev2_port: 22 + + # Temp IP we add to DEV1 so it can reach DEV2 + dev2_side_ip: "192.168.1.11/24" + dev1_iface: "br-wan" + + # DEV2 behind the tunnel + dev2_ssh_user: "root" + dev2_passfiles: + - "basicpass" + - "basicpass2" + + # SSH options used from controller + ssh_opts_common: "-o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o NumberOfPasswordPrompts=1 -o ConnectTimeout=15" + + # Image to stage on DEV2 noninvasive we do not run update -w here + image_filename: "fox200-2.2.1-r6801.bin" + image_md5: "56b7211709de617e058b98d4204e2562" + # Optional SHA256; leave empty to skip SHA256 checks + image_sha256: "" + dev2_image_dir: "/tmp" + dev2_image_path: "{{ dev2_image_dir }}/{{ image_filename }}" + + # Reboot delay in minutes integer string if set we will schedule reboot on DEV2 + rebootin: "" + + # ---------------- RabbitMQ journaling (mirrors rebootin222 style) ---------------- + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + + + pre_tasks: + # ------------------------------- Hostname sanity DEV1 ------------------------------- + - name: Read DEV1 hostname from env busybox safe + ansible.builtin.raw: > + {{ pathprefix }} + (cat /proc/sys/kernel/hostname 2>/dev/null || echo "") + register: dev1_host_read + changed_when: false + + - name: Stop if DEV1 connected hostname differs from inventory + ansible.builtin.meta: end_host + when: (dev1_host_read.stdout | trim | length > 0) and + ((dev1_host_read.stdout | trim) != (inventory_hostname | string)) + + tasks: + # ---------------------------- Idempotent temp IP on DEV1 ---------------------------- + - name: Add temporary IP on DEV1 idempotent treat File exists as OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a add {{ dev2_side_ip }} dev {{ dev1_iface }} + register: add_ip + changed_when: add_ip.rc == 0 + failed_when: > + add_ip.rc != 0 + and ('File exists' not in (add_ip.stdout | default(''))) + and ('File exists' not in (add_ip.stderr | default(''))) + + - name: Debug add ip results + ansible.builtin.debug: + msg: + - "add_ip.rc={{ add_ip.rc | default('') }}" + - "add_ip.stdout={{ (add_ip.stdout | default('')) | trim }}" + - "add_ip.stderr={{ (add_ip.stderr | default('')) | trim }}" + + # ---------------------------- Discover MAC via bridge FDB and add static ARP on DEV1 ---------------------------- + - name: Read dynamic MAC on eth0 behind {{ dev1_iface }} via bridge fdb + ansible.builtin.raw: > + {{ pathprefix }} + bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1 + register: dev2_mac_scan + changed_when: false + + - name: Set dev2_mac from fdb scan (if any) + ansible.builtin.set_fact: + dev2_mac: "{{ (dev2_mac_scan.stdout | default('') ) | trim }}" + changed_when: false + + - name: Remove current ARP entry for DEV2 on DEV1 (best-effort) + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true + register: dev2_arp_del + changed_when: false + failed_when: false + + - name: Add static ARP for DEV2 on DEV1 + ansible.builtin.raw: > + {{ pathprefix }} + ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent + register: dev2_arp_add + changed_when: dev2_arp_add.rc == 0 + failed_when: > + dev2_arp_add.rc != 0 + and ('File exists' not in (dev2_arp_add.stdout | default(''))) + and ('File exists' not in (dev2_arp_add.stderr | default(''))) + + - name: Debug static ARP result + ansible.builtin.debug: + msg: + - "dev2_mac={{ dev2_mac | default('UNSET') }}" + - "arp_add.rc={{ dev2_arp_add.rc | default('') }}" + - "arp_add.out={{ (dev2_arp_add.stdout | default('')) | trim }}" + - "arp_add.err={{ (dev2_arp_add.stderr | default('')) | trim }}" + + - name: Skip ARP add because MAC not found + when: dev2_mac is not defined or dev2_mac | length == 0 + ansible.builtin.debug: + msg: "No suitable dynamic MAC found via bridge fdb; skipping static ARP add on DEV1" + + # ---------------------------- Local tunnel preparation ---------------------------- + - name: Pick a free local port for the tunnel controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + pick() { + for i in $(seq 1 25); do + p="$(shuf -i 20000-39999 -n 1)" + if command -v ss >/dev/null 2>&1; then + if ! ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$"; then + echo "$p"; return 0 + fi + else + if ! nc -z 127.0.0.1 "$p" >/dev/null 2>&1; then + echo "$p"; return 0 + fi + fi + done + return 1 + } + pick + register: pick_port + changed_when: false + + - name: Stop if no free local port + ansible.builtin.meta: end_host + when: (pick_port.stdout | trim | length) == 0 + + - name: Record chosen local port and control directory + delegate_to: localhost + ansible.builtin.set_fact: + _local_port: "{{ pick_port.stdout | trim }}" + _ctrl_dir: "{{ lookup('ansible.builtin.pipe', 'mktemp -d') }}" + + - name: Build tunnel control socket path + delegate_to: localhost + ansible.builtin.set_fact: + _ctrl_sock: "{{ _ctrl_dir }}/ssh_tunnel_ctl" + + - name: Debug chosen tunnel params + delegate_to: localhost + ansible.builtin.debug: + msg: + - "local_port={{ _local_port }}" + - "ctrl_dir={{ _ctrl_dir }}" + - "ctrl_sock={{ _ctrl_sock }}" + + - name: issue arping with own IP + ansible.builtin.raw: arping -U -I eth0 192.168.1.11 -c 3 + + # ---------------------------- Start SSH local forward via DEV1 ---------------------------- + - name: Start SSH tunnel via DEV1 background ControlMaster + delegate_to: localhost + ansible.builtin.shell: | + set -e + USER="{{ dev1_user }}" + HOST="{{ ansible_host | default(inventory_hostname) }}" + sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \ + -M -S "{{ _ctrl_sock }}" \ + -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \ + "${USER}@${HOST}" + args: + executable: /bin/bash + register: start_tunnel + changed_when: true + + - name: Small delay for tunnel to settle + delegate_to: localhost + ansible.builtin.wait_for: + timeout: 1 + changed_when: false + + - name: Verify tunnel master running proper O check with destination + delegate_to: localhost + ansible.builtin.shell: | + set -e + HOST="{{ ansible_host | default(inventory_hostname) }}" + ssh -S "{{ _ctrl_sock }}" -O check "{{ dev1_user }}@${HOST}" 2>&1 || true + register: tun_check + changed_when: false + + - name: Debug tunnel check + delegate_to: localhost + ansible.builtin.debug: + msg: + - "tunnel_check.rc={{ tun_check.rc }}" + - "tunnel_check.out={{ (tun_check.stdout | default('')) | trim }}" + + # ---------------------------- Controller-side sanity for DEV2 auth ---------------------------- + - name: Sanity confirm tunnel TCP reachability to DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + nc -z -w5 127.0.0.1 "{{ _local_port }}" + register: nc_probe + changed_when: false + ignore_errors: true + + - name: Debug reachability result + delegate_to: localhost + ansible.builtin.debug: + msg: + - "nc.rc={{ nc_probe.rc }}" + - "nc.stdout={{ (nc_probe.stdout | default('')) | trim }}" + - "nc.stderr={{ (nc_probe.stderr | default('')) | trim }}" + + - name: Stop if tunnel TCP check failed + ansible.builtin.meta: end_host + when: nc_probe.rc != 0 + + - name: Sanity show presence and permissions of passfiles on controller + delegate_to: localhost + ansible.builtin.shell: | + set -e + ls -l basicpass basicpass2 2>/dev/null || echo "no passfiles in CWD" + register: dev2_ls + changed_when: false + + - name: Debug passfiles listing + delegate_to: localhost + ansible.builtin.debug: + msg: + - "{{ (dev2_ls.stdout | default('')) | trim }}" + - "{{ (dev2_ls.stderr | default('')) | trim }}" + + # ---------------------------- Pick DEV2 password root only ---------------------------- + - name: Try DEV2 login with basicpass root + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass + changed_when: false + ignore_errors: true + + - name: Select basicpass as working passfile + when: dev2_try_basicpass.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass" + changed_when: false + + - name: Try DEV2 login with basicpass2 root only if first failed + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f basicpass2 ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" root@127.0.0.1 echo OK >/dev/null 2>&1 + register: dev2_try_basicpass2 + changed_when: false + ignore_errors: true + + - name: Select basicpass2 as working passfile + when: dev2_passfile_used is not defined and dev2_try_basicpass2.rc == 0 + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "basicpass2" + changed_when: false + + - name: Set dev2_passfile_used to NONE if neither worked + when: dev2_passfile_used is not defined + delegate_to: localhost + ansible.builtin.set_fact: + dev2_passfile_used: "NONE" + changed_when: false + + - name: Debug selected DEV2 passfile or NONE + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "try_basicpass.rc={{ (dev2_try_basicpass.rc | default('NA')) }}" + - "try_basicpass2.rc={{ (dev2_try_basicpass2.rc | default('SKIPPED')) }}" + + - name: Read DEV2 hostname via tunnel (busybox-safe) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /proc/sys/kernel/hostname 2>/dev/null || hostname || echo" + args: + executable: /bin/bash + register: dev2_host_read + changed_when: false + + - name: Normalize hostnames for strict compare + ansible.builtin.set_fact: + _inv_hn: "{{ (inventory_hostname | string) | trim | regex_replace('\\r+$','') | lower }}" + _dev1_hn: "{{ (dev1_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + _dev2_hn: "{{ (dev2_host_read.stdout | default('')) | trim | regex_replace('\\r+$','') | lower }}" + + - name: Debug hostname bytes (hex) + delegate_to: localhost + ansible.builtin.debug: + msg: + - "inv={{ _inv_hn | tojson }}" + - "dev1={{ _dev1_hn | tojson }}" + - "dev2={{ _dev2_hn | tojson }}" + + # ---------------- Hostname equality guard (soft-journal and stop) ---------------- + - name: Enforce DEV2 hostname equals inventory and DEV1 + block: + - name: Fail if DEV2 hostname differs from inventory/DEV1 + ansible.builtin.fail: + msg: > + Hostname mismatch: DEV2='{{ _dev2_hn }}', + inventory='{{ _inv_hn }}', + DEV1='{{ _dev1_hn }}' + when: (_dev2_hn != _inv_hn) or (_dev2_hn != _dev1_hn) + rescue: + - name: Initialize journal array for hostname mismatch + ansible.builtin.set_fact: + _journal: [] + _blocked: true + _prep_blocked: false + delegate_to: localhost + + - name: Append hostname mismatch to journal + ansible.builtin.set_fact: + _journal: "{{ _journal + [ 'Hostname mismatch: DEV2=' ~ _dev2_hn ~ ', inventory=' ~ _inv_hn ~ ', DEV1=' ~ _dev1_hn ] }}" + delegate_to: localhost + + - name: Build control queue payload for indoor aborted journal (hostname) + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue (hostname) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_hn_resp + changed_when: (rmq_journal_indoor_aborted_hn_resp.json is defined) and (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_hn_resp.status != 200) or + (rmq_journal_indoor_aborted_hn_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_hn_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop host after hostname mismatch + ansible.builtin.meta: end_host + + # ---------------------------- SOFT-FAIL JOURNAL INIT + PREP MARKER CHECK ---------------------------- + - name: Initialize journal and flags + ansible.builtin.set_fact: + _journal: [] + _prep_blocked: false + _blocked: false + + - name: Build specific marker filename (no .bin) + ansible.builtin.set_fact: + _marker_specific: "/tmp/prepared_for_{{ image_filename | regex_replace('\\.bin$','') }}" + + - name: Check for any preparation markers on DEV2 (wildcard count) + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "ls /tmp/prepared_for* 2>/dev/null | wc -l" + args: + executable: /bin/bash + register: dev2_prep_count + changed_when: false + ignore_errors: true + + - name: Soft-block if any prep markers already present + when: dev2_prep_count is defined and (dev2_prep_count.stdout is defined) and ((dev2_prep_count.stdout | trim | int) > 0) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}" + + # ---------------------------- DEV2 version firmux primary check ---------------------------- + - name: Read DEV2 firmux release + when: dev2_passfile_used != "NONE" + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "cat /usr/lib/release/firmux 2>/dev/null || true" + args: + executable: /bin/bash + register: dev2_firmux + changed_when: false + ignore_errors: true + + - name: Debug firmux content DEV2 + when: dev2_firmux is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "DEV2 firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + + # ---------------------------- Normalize rebootin early ---------------------------- + - name: "Normalize rebootin (phase 1: raw/is_now/is_int/minutes)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_raw: "{{ rebootin | default('') | string | trim | lower }}" + _reboot_is_now: "{{ (rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate'] }}" + _reboot_is_int: "{{ (rebootin | default('') | string | trim) is match('^\\d+$') }}" + _reboot_minutes: >- + {{ + 0 if ((rebootin | default('') | string | trim | lower) in ['', 'now', 'immediate']) + else (rebootin | int if ((rebootin | default('') | string | trim) is match('^\\d+$') ) else -1) + }} + + - name: "Normalize rebootin (phase 2: requested flag)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_requested: "{{ _reboot_is_now or _reboot_is_int }}" + + # >>> NEW: compute effective seconds (0 min → 20 seconds) + - name: "Normalize rebootin (phase 3: compute seconds)" + delegate_to: localhost + ansible.builtin.set_fact: + _reboot_seconds: >- + {{ + 20 if (_reboot_requested | default(false)) and ((_reboot_minutes | int) == 0) + else ((_reboot_minutes | int) * 60) + }} + + # ---------------------------- Journal: indoor start (after we know we can proceed) ---------------------------- + - name: Build control queue payload for indoor start journal + ansible.builtin.set_fact: + journal_indoor_start: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 is reachable, starting with update. + image={{ image_filename }}, md5={{ image_md5 }}, + firmux={{ (dev2_firmux.stdout | default('unknown')) | trim }}, + reboot={{ 'in ' ~ (_reboot_seconds | int) ~ ' seconds' if (_reboot_is_now and (_reboot_minutes | int) == 0) else (rebootin | default('') | trim) }} + when: dev2_passfile_used != "NONE" + delegate_to: localhost + + - name: Publish indoor start journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_start | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_start_resp + changed_when: (rmq_journal_indoor_start_resp.json is defined) and (rmq_journal_indoor_start_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_start_resp.status != 200) or + (rmq_journal_indoor_start_resp.json is not defined) or + (not (rmq_journal_indoor_start_resp.json.routed | default(false) | bool)) + when: journal_indoor_start is defined + delegate_to: localhost + + # ---------------------------- Stage image on DEV2 no write ---------------------------- + - name: Check local presence of image file + delegate_to: localhost + ansible.builtin.stat: + path: "{{ image_filename }}" + register: local_img + + - name: Soft-block if image missing locally + when: not local_img.stat.exists + ansible.builtin.set_fact: + _blocked: true + _journal: "{{ _journal + [ 'Local image missing on controller: ' ~ image_filename ] }}" + + - name: Compute local md5 of the image + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.shell: | + set -e + md5sum "{{ image_filename }}" | awk '{print $1}' + register: local_md5 + changed_when: false + + - name: Verify local md5 matches expected + when: local_img.stat.exists + delegate_to: localhost + ansible.builtin.assert: + that: + - (local_md5.stdout | trim) == image_md5 + fail_msg: "Local md5 does not match expected got {{ local_md5.stdout | trim }} expected {{ image_md5 }}" + success_msg: "Local md5 matches expected" + + # Optional SHA256 local + - name: Compute local sha256 of the image (if provided) + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + sha256sum "{{ image_filename }}" | awk '{print $1}' + register: local_sha256 + changed_when: false + ignore_errors: true + + - name: Soft-block if local sha256 mismatch or unavailable + when: local_img.stat.exists and (image_sha256 | default('') | length) > 0 and (local_sha256 is not defined or (local_sha256.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Local sha256 mismatch/unavailable: have=' ~ ((local_sha256.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + # fw_printenv health before upload (soft-fail) + - name: Read fw_printenv size (line count) on DEV2 + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "(fw_printenv 2>/dev/null | wc -l) || echo 0" + args: + executable: /bin/bash + register: dev2_fwenv_wc + changed_when: false + ignore_errors: true + + - name: Soft-block if fw_printenv too small + when: dev2_fwenv_wc is defined and (dev2_fwenv_wc.stdout is defined) and ((dev2_fwenv_wc.stdout | trim | int) < 30) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'fw_printenv too small on DEV2: ' ~ (dev2_fwenv_wc.stdout | trim) ~ ' lines (<30). Skipping image staging' ] }}" + + - name: Check existing image md5 on DEV2 NOFILE if absent + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "[ -f '{{ dev2_image_path }}' ] && md5sum '{{ dev2_image_path }}' | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_before + changed_when: false + + - name: Copy image to DEV2 via tunnel only if missing or md5 mismatch + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_before.stdout | trim) != image_md5) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" scp \ + -P "$PORT" \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "{{ image_filename }}" "{{ dev2_ssh_user }}@127.0.0.1:{{ dev2_image_dir }}/" + args: + executable: /bin/bash + register: scp_push + changed_when: true + + - name: Compute md5 of image on DEV2 after copy or if exists + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "md5sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOFILE" + args: + executable: /bin/bash + register: dev2_md5_after + changed_when: false + + - name: Verify DEV2 md5 matches expected (soft-fail journal) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and ((dev2_md5_after.stdout | trim) != image_md5) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote md5 mismatch on DEV2: have=' ~ (dev2_md5_after.stdout | trim) ~ ' expected=' ~ image_md5 ~ '. Skipping further prep' ] }}" + + # Optional SHA256 remote + - name: Compute sha256 of image on DEV2 (if provided) + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "sha256sum '{{ dev2_image_path }}' 2>/dev/null | awk '{print \$1}' || echo NOSHA" + args: + executable: /bin/bash + register: dev2_sha256_after + changed_when: false + ignore_errors: true + + - name: Soft-block if DEV2 sha256 mismatch/unavailable + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) and (image_sha256 | default('') | length) > 0 and (dev2_sha256_after is not defined or (dev2_sha256_after.stdout | trim) != (image_sha256 | trim)) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'Remote sha256 mismatch/unavailable on DEV2: have=' ~ ((dev2_sha256_after.stdout | default('NA')) | trim) ~ ' expected=' ~ (image_sha256 | trim) ] }}" + + - name: Run noninvasive update check on DEV2 update c + when: local_img.stat.exists and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=15 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -c '{{ dev2_image_path }}' 2>&1 || true" + args: + executable: /bin/bash + register: dev2_update_check + changed_when: false + + - name: Debug update c output DEV2 + when: dev2_update_check is defined + delegate_to: localhost + ansible.builtin.debug: + msg: "{{ (dev2_update_check.stdout | default('')) | trim }}" + + - name: Soft-block if update -c did not say valid + when: dev2_update_check is defined and not ((dev2_update_check.stdout | default('') | lower) is search('valid')) + ansible.builtin.set_fact: + _prep_blocked: true + _blocked: true + _journal: "{{ _journal + [ 'update -c did not return valid on DEV2; output=' ~ ((dev2_update_check.stdout | default('')) | trim) ] }}" + + # ---------------------------- Journal: indoor aborted (if any blockers) ---------------------------- + - name: Build control queue payload for indoor aborted journal + ansible.builtin.set_fact: + journal_indoor_aborted: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + indoor: update aborted with following reason(s): {{ (_journal | default([])) | join('; ') }} + when: (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor aborted journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_aborted | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_aborted_resp + changed_when: (rmq_journal_indoor_aborted_resp.json is defined) and (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_aborted_resp.status != 200) or + (rmq_journal_indoor_aborted_resp.json is not defined) or + (not (rmq_journal_indoor_aborted_resp.json.routed | default(false) | bool)) + when: journal_indoor_aborted is defined + delegate_to: localhost + + # ============================ ACTUAL UPGRADE WRITE + BANK FLIP (only if not blocked) ============================ + - name: Upgrade write and flip block + when: not (_blocked | default(false)) + block: + - name: Write image on DEV2 (this will take a while) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=0 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "update -w '{{ dev2_image_path }}'" + args: + executable: /bin/bash + register: dev2_up_write + changed_when: true + failed_when: dev2_up_write.stdout is not search('update is complete') + + - name: Read current active partition on DEV2 + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_before + changed_when: false + failed_when: (dev2_active_before.stdout | trim) not in ['1','2'] + + - name: Determine new active value for DEV2 + when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2']) + ansible.builtin.set_fact: + dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ dev2_new_active }} + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_setenv active {{ dev2_new_active }}" + args: + executable: /bin/bash + register: dev2_setenv_out + changed_when: true + + - name: Verify active partition flipped on DEV2 + when: dev2_new_active is defined + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "fw_printenv active | awk -F= '/^active=/{print \$2}'" + args: + executable: /bin/bash + register: dev2_active_after + changed_when: false + failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string) + + - name: Create specific prep marker on DEV2 (no generic) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \ + "touch '{{ _marker_specific }}'" + args: + executable: /bin/bash + register: dev2_marker_write + changed_when: true + ignore_errors: true + + # ---------------------------- Reboot scheduling (normalized) ---------------------------- + - name: Schedule delayed reboot on DEV2 minutes + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + ansible.builtin.shell: | + set -e + PORT="{{ _local_port }}" + SECS="{{ _reboot_seconds | int }}" + CMD='/sbin/reboot -d '"${SECS}"' >/dev/null 2>&1 &' + sshpass -f "{{ dev2_passfile_used }}" ssh \ + -o AddressFamily=inet \ + -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \ + -o ConnectTimeout=10 \ + -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" "${CMD}" + args: + executable: /bin/bash + register: dev2_reboot_sched + changed_when: true + ignore_errors: true + + - name: Build control queue payload for indoor-restart-scheduled tag + ansible.builtin.set_fact: + tag_restart_sched_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "indoor-restart-scheduled" + when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false)) + delegate_to: localhost + + - name: Publish indoor-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_sched_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_sched_resp + changed_when: (rmq_tag_restart_sched_resp.json is defined) and (rmq_tag_restart_sched_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_sched_resp.status != 200) or + (rmq_tag_restart_sched_resp.json is not defined) or + (not (rmq_tag_restart_sched_resp.json.routed | default(false) | bool)) + when: tag_restart_sched_payload is defined + delegate_to: localhost + + + - name: Note reboot request bad format + when: (rebootin | default('') | string | trim | length) > 0 and not _reboot_requested + ansible.builtin.debug: + msg: "Reboot requested but value '{{ rebootin | string | trim }}' is invalid; not applied" + + # ---------------------------- Journal: indoor updated and reboot schedule ---------------------------- + - name: Build control queue payload for indoor updated journal + ansible.builtin.set_fact: + _write_success: "{{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else false }}" + when: not (_blocked | default(false)) + delegate_to: localhost + + - name: Build indoor updated journal payload text + ansible.builtin.set_fact: + journal_indoor_updated: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Indoor: Dev2 updated, scheduling reboot {{ + ('in ' ~ (_reboot_seconds | int) ~ ' seconds') if (_reboot_requested and (_reboot_minutes | int) == 0) + else ('in ' ~ (_reboot_minutes | int) ~ ' minutes') if (_reboot_requested and (_reboot_minutes | int) > 0) + else 'not requested' + }}. + write_done={{ _write_success }}, + active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}, + active_after={{ (dev2_active_after.stdout | default('NA')) | trim }} + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Publish indoor updated journal to control queue + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_indoor_updated | to_json }}" + payload_encoding: "string" + register: rmq_journal_indoor_updated_resp + changed_when: (rmq_journal_indoor_updated_resp.json is defined) and (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_indoor_updated_resp.status != 200) or + (rmq_journal_indoor_updated_resp.json is not defined) or + (not (rmq_journal_indoor_updated_resp.json.routed | default(false) | bool)) + when: journal_indoor_updated is defined + delegate_to: localhost + + + - name: Init after-upgrade scheduling vars (attempt 1 + delay from reboot + 6m) + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + au_delay_sec: "{{ (_reboot_seconds | int) + 360 }}" + when: not (_blocked | default(false)) and (_write_success | bool) + delegate_to: localhost + + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: au_delay_sec is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_indoor_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr | default(image_filename | regex_replace('\\.bin$','')) }}" + attempt: "{{ au_attempt }}" + max_attempts: "{{ au_max_attempts }}" + current_delay_sec: "{{ au_delay_sec }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: au_delay_sec is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + + # ---------------------------- summary ---------------------------- + - name: Summary show key results + delegate_to: localhost + ansible.builtin.debug: + msg: + - "dev2_passfile_used={{ dev2_passfile_used }}" + - "dev2_firmux={{ (dev2_firmux.stdout | default('')) | trim }}" + - "local_image_present={{ local_img.stat.exists | default(false) }}" + - "local_md5={{ (local_md5.stdout | default('NA')) | trim }}" + - "dev2_md5_before={{ (dev2_md5_before.stdout | default('NA')) | trim }}" + - "dev2_md5_after={{ (dev2_md5_after.stdout | default('NA')) | trim }}" + - "update_c_len={{ (dev2_update_check.stdout | default('') ) | length }}" + - "write_done={{ (dev2_up_write.stdout | default('')) is search('update is complete') if (dev2_up_write is defined) else 'NA' }}" + - "active_before={{ (dev2_active_before.stdout | default('NA')) | trim }}" + - "active_after={{ (dev2_active_after.stdout | default('NA')) | trim }}" + - "reboot_requested={{ _reboot_requested | default(false) }}" + - "reboot_delay_minutes={{ (_reboot_minutes | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_delay_seconds={{ (_reboot_seconds | int) if (_reboot_requested | default(false)) else 'NA' }}" + - "reboot_applied={{ (dev2_reboot_sched is defined and dev2_reboot_sched.rc is defined and dev2_reboot_sched.rc == 0) | default(false) }}" + - "prep_blocked={{ _prep_blocked | default(false) }}" + - "blocked={{ _blocked | default(false) }}" + - "journal={{ (_journal | default([])) | join(' || ') }}" + + post_tasks: + - name: Cleanup always + block: + - ansible.builtin.debug: + msg: "entering cleanup block" + changed_when: false + delegate_to: localhost + always: + - name: Close tunnel best effort + delegate_to: localhost + ansible.builtin.shell: | + ssh -S "{{ _ctrl_sock | default('/dev/null') }}" -O exit 2>/dev/null || true + changed_when: false + ignore_errors: true + + - name: Remove control dir best effort + delegate_to: localhost + ansible.builtin.file: + path: "{{ _ctrl_dir | default('/tmp/none') }}" + state: absent + ignore_errors: true + + - name: Remove temporary IP on DEV1 idempotent Cannot assign requested address OK + ansible.builtin.raw: > + {{ pathprefix }} + ip a del {{ dev2_side_ip }} dev {{ dev1_iface }} + register: del_ip + changed_when: del_ip.rc == 0 + failed_when: > + del_ip.rc != 0 + and ('Cannot assign requested address' not in (del_ip.stdout | default(''))) + and ('Cannot assign requested address' not in (del_ip.stderr | default(''))) + + - name: Debug del ip results + ansible.builtin.debug: + msg: + - "del_ip.rc={{ del_ip.rc | default('') }}" + - "del_ip.stdout={{ (del_ip.stdout | default('')) | trim }}" + - "del_ip.stderr={{ (del_ip.stderr | default('')) | trim }}" + when: del_ip is defined diff --git a/files/ansible-playbooks/update-reboot.yml b/files/ansible-playbooks/update-reboot.yml new file mode 100644 index 0000000..9b72587 --- /dev/null +++ b/files/ansible-playbooks/update-reboot.yml @@ -0,0 +1,4 @@ +--- +# update-reboot.yml — thin wrapper, no var forwarding. +# Delegates entirely to the unified updater. +- import_playbook: update-rebootin222.yml diff --git a/files/ansible-playbooks/update-rebootin222.yml b/files/ansible-playbooks/update-rebootin222.yml new file mode 100644 index 0000000..5660453 --- /dev/null +++ b/files/ansible-playbooks/update-rebootin222.yml @@ -0,0 +1,831 @@ +--- +- name: Upgrade firmware safely (no Python on target) + hosts: all + gather_facts: no + + vars: + # RabbitMQ (pull from env if provided) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}" + + # NEW: Post-upgrade check scheduling (via holding queue -> DLX) + # The holding queue is bound to exchange 'deviceconfig.holding' with routing key 'deviceconfig'. + # Messages published here carry a per-message TTL (AMQP 'expiration' property, in ms). + # Once TTL elapses, messages dead-letter to exchange 'deviceconfig' with same routing key, + # where a consumer will perform the after-upgrade verification (attempt-based backoff lives on consumer side). + afterupgrade_hold_exchange: "{{ lookup('env','AFTERUP_HOLD_EXCHANGE') | default('deviceconfig.holding', true) }}" + afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}" + # Queue name is not used for publish; present for documentation/reference only + afterupgrade_hold_queue: "{{ lookup('env','AFTERUP_HOLD_QUEUE') | default('queue_deviceconfig_holdingzone', true) }}" + + # REQUIRED (pass via -e) + firmware_path: /tmp/2.2.2-r9778.bin + firmware_sha256: "15bc6f3492321196bb5220014c4441a4dceab2af5f570c46a23140cd37b65a02" + + # Helper computed vars + fw_base: "{{ firmware_path | basename }}" + fw_name: "{{ fw_base | regex_replace('\\.bin$', '') }}" + fw_banner_repr: "{{ fw_name | regex_replace('-r', ' rev ') }}" + fw_marker: "/tmp/prepared_for_{{ fw_name }}" + pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; " + + tasks: + + # ----------------------------- HOSTNAME PREFLIGHT ----------------------------- + - name: Hostname preflight + block: + - name: Read remote HOSTNAME + ansible.builtin.raw: "{{ pathprefix }} echo \"$HOSTNAME\"" + register: host_env + changed_when: false + + - name: Debug hostnames + ansible.builtin.debug: + msg: + - "remote_hostname={{ host_env.stdout | trim }}" + - "inventory_hostname={{ inventory_hostname }}" + + - name: Stop if connected hostname differs from inventory + ansible.builtin.fail: + msg: "Aborting: connected host reported hostname '{{ host_env.stdout | trim }}' which differs from inventory '{{ inventory_hostname }}'." + when: (host_env.stdout | trim) != inventory_hostname + + rescue: + - name: Build failure task name and detail (hostname preflight) + ansible.builtin.set_fact: + fail_task_name: "{{ ansible_failed_task.name | default('hostname preflight') }}" + fail_detail_raw: >- + {{ ansible_failed_result.msg + | default(ansible_failed_result.stderr) + | default(ansible_failed_result.stdout) + | default('no additional error output') + | trim }} + + - name: Build failure summary text (hostname preflight) + ansible.builtin.set_fact: + fail_summary: >- + Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }} + + - name: Truncate failure summary to ~400 chars (hostname preflight) + ansible.builtin.set_fact: + fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}" + + - name: Build control queue payload for failure journal (hostname preflight) + ansible.builtin.set_fact: + journal_failure_payload_pre: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "{{ fail_summary_short }}" + + - name: Publish failure journal to control queue (hostname preflight) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_failure_payload_pre | to_json }}" + payload_encoding: "string" + register: rmq_journal_pre_resp + changed_when: (rmq_journal_pre_resp.json is defined) and (rmq_journal_pre_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_pre_resp.status != 200) or + (rmq_journal_pre_resp.json is not defined) or + (not (rmq_journal_pre_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Build control queue payload for update-aborted tag (hostname preflight) + ansible.builtin.set_fact: + tag_failed_payload_pre: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "update-aborted" + + - name: Publish update-aborted tag to control queue (hostname preflight) + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_failed_payload_pre | to_json }}" + payload_encoding: "string" + register: rmq_tag_failed_pre_resp + changed_when: (rmq_tag_failed_pre_resp.json is defined) and (rmq_tag_failed_pre_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_failed_pre_resp.status != 200) or + (rmq_tag_failed_pre_resp.json is not defined) or + (not (rmq_tag_failed_pre_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Stop play after hostname preflight failure + ansible.builtin.meta: end_play + + # --- Tag device as update-in-progress at start --- + - name: Build control queue payload for update-in-progress tag + ansible.builtin.set_fact: + tag_inprogress_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "update-in-progress" + + - name: Publish update-in-progress tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_inprogress_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_inprogress_resp + changed_when: (rmq_tag_inprogress_resp.json is defined) and (rmq_tag_inprogress_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_inprogress_resp.status != 200) or + (rmq_tag_inprogress_resp.json is not defined) or + (not (rmq_tag_inprogress_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Log control queue tag publish result + ansible.builtin.debug: + var: rmq_tag_inprogress_resp.json + when: rmq_tag_inprogress_resp is defined + + # --------------------- Prepared marker check BEFORE SSID scan ----------------- + - name: Check if any prepared marker exists + ansible.builtin.raw: "{{ pathprefix }} [ -e /tmp/prepared_for* ] && echo PRESENT || echo ABSENT" + register: prep_scan + changed_when: false + + - name: Debug marker presence + ansible.builtin.debug: + msg: "prepared_marker={{ prep_scan.stdout | trim }}" + + - name: Journal preparation already present, skipping update steps + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: >- + {{ + { + "inscope_device": (ansible_hostname | default(inventory_hostname)), + "task_name": "journal_add", + "task_result": ( + "Preparation already present for " ~ fw_banner_repr ~ + "; marker " ~ fw_marker ~ + ". Skipping update steps." + ) + } | to_json + }} + payload_encoding: "string" + register: rmq_journal_prep_present + changed_when: (rmq_journal_prep_present.json is defined) and (rmq_journal_prep_present.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_prep_present.status != 200) or + (rmq_journal_prep_present.json is not defined) or + (not (rmq_journal_prep_present.json.routed | default(false) | bool)) + delegate_to: localhost + when: (prep_scan.stdout | trim) == 'PRESENT' + + + - name: End play for this host (already prepared) + ansible.builtin.meta: end_host + when: (prep_scan.stdout | trim) == 'PRESENT' + + # --- SSID scan & journal (does not stop the play) --- + - name: Count SSID lines in /tmp/config.json (filtered) + ansible.builtin.raw: > + {{ pathprefix }} + grep '"ssid"' /tmp/config.json 2>/dev/null | grep -vE '\{|SC|auto|backha' | wc -l + register: ssid_lines + changed_when: false + + - name: Debug SSID count + + ansible.builtin.debug: + msg: "ssid_count={{ (ssid_lines.stdout | default('0')) | trim }}" + + - name: Build joined SSID list when multiple SSIDs found (≥3) + ansible.builtin.raw: > + {{ pathprefix }} + grep '"ssid"' /tmp/config.json | grep -vE '\{|SC|auto|backha' \ + | sed -E 's/.*"ssid": "([^"]+)".*/\1/' \ + | awk 'NR==1 { out=$0; next } { out=out","$0 } END { print out }' + register: ssid_concat + changed_when: false + when: (ssid_lines.stdout | trim | int) >= 3 + + - name: Build control queue payload for SSID journal (journal_add) + ansible.builtin.set_fact: + ssid_journal_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "Multiple SSID! {{ ssid_concat.stdout | trim }}" + when: (ssid_lines.stdout | trim | int) >= 3 + + - name: Publish SSID journal to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ ssid_journal_payload | to_json }}" + payload_encoding: "string" + register: rmq_ssid_journal_resp + changed_when: (rmq_ssid_journal_resp.json is defined) and (rmq_ssid_journal_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_ssid_journal_resp.status != 200) or + (rmq_ssid_journal_resp.json is not defined) or + (not (rmq_ssid_journal_resp.json.routed | default(false) | bool)) + delegate_to: localhost + when: ssid_journal_payload is defined + + # ----------------------------- MAIN UPDATE BLOCK ----------------------------- + - name: Firmware update main block + block: + + - name: Check if firmware image is already on the device + ansible.builtin.raw: "{{ pathprefix }} [ -f '{{ firmware_path }}' ] && echo OK || echo MISSING" + register: fw_exists + changed_when: false + + - name: Count fw_printenv lines + ansible.builtin.raw: "{{ pathprefix }} fw_printenv 2>/dev/null | wc -l" + register: env_line_count + changed_when: false + + - name: Debug fw_printenv line count + ansible.builtin.debug: + msg: "fw_printenv_lines={{ env_line_count.stdout | trim }}" + + - name: Stop if bootloader environment looks invalid (<7 lines) + ansible.builtin.fail: + msg: "Aborting: fw_printenv returned only {{ env_line_count.stdout | trim }} lines (<7) — environment missing or corrupted." + when: (env_line_count.stdout | trim | int) < 7 + + - name: Read first line of /etc/banner (current running version) + ansible.builtin.raw: "{{ pathprefix }} cat /etc/banner | grep -i rev | head -n1" + register: banner + changed_when: false + + - name: current version + ansible.builtin.debug: + msg: + - "current banner: {{ banner.stdout | trim }}" + + - name: Stop if target version matches current (/etc/banner already at {{ fw_banner_repr }}) + ansible.builtin.fail: + msg: "Aborting: device already runs {{ fw_banner_repr }} (banner: {{ banner.stdout | trim }})" + when: banner.stdout is search(fw_banner_repr) + + - name: Upload firmware to /tmp via scp (controller-side) + ansible.builtin.shell: | + set -e + SRC='{{ fw_src_local | default("/opt/containers/ansible-worker/app/2.2.2-r9778.bin") }}' + DST_USER='{{ ansible_user | default("root") }}' + DST_HOST='{{ ansible_host | default(inventory_hostname) }}' + test -f "$SRC" + sshpass -p '{{ ansible_ssh_pass }}' scp -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \ + "$SRC" "${DST_USER}@${DST_HOST}:{{ firmware_path }}" + delegate_to: localhost + when: fw_exists.stdout is not defined or (fw_exists.stdout | trim) != 'OK' + changed_when: true + + - name: Re-check firmware presence after optional upload + ansible.builtin.raw: "{{ pathprefix }} test -f '{{ firmware_path }}' && echo OK || echo MISSING" + register: fw_exists2 + changed_when: false + failed_when: (fw_exists2.stdout | trim) != 'OK' + + - name: Compute sha256 of the uploaded image + ansible.builtin.raw: "{{ pathprefix }} sha256sum '{{ firmware_path }}' | awk '{print $1}'" + register: sha_out + changed_when: false + + - name: Verify sha256 matches expected + ansible.builtin.fail: + msg: "SHA256 mismatch for {{ firmware_path }}. Got {{ sha_out.stdout | trim }}, expected {{ firmware_sha256 }}" + when: (sha_out.stdout | trim) != (firmware_sha256 | trim) + + - name: sha256 verification debug + ansible.builtin.debug: + msg: + - "sha256sum is: {{ sha_out.stdout | trim }}" + + - name: Check image validity (update -c must say 'valid') + ansible.builtin.raw: "{{ pathprefix }} update -c '{{ firmware_path }}'" + register: up_check + changed_when: false + failed_when: up_check.stdout.strip() != 'valid' + + - name: image verification debug + ansible.builtin.debug: + msg: + - ".bin verification is: {{ up_check.stdout | trim }}" + + # - name: forced stop before writing + # ansible.builtin.meta: end_play + + - name: Write image (this will take a while) + ansible.builtin.raw: "{{ pathprefix }} update -w '{{ firmware_path }}'" + register: up_write + changed_when: true + failed_when: up_write.stdout is not search('update is complete') + + - name: Read current active partition + ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'" + register: active_before + changed_when: false + failed_when: active_before.stdout | trim not in ['1','2'] + + - name: Determine new active value + ansible.builtin.set_fact: + new_active: "{{ '1' if (active_before.stdout | trim) == '2' else '2' }}" + + - name: Switch active partition to {{ new_active }} + ansible.builtin.raw: "{{ pathprefix }} fw_setenv active {{ new_active }}" + register: setenv_out + changed_when: true + + - name: Verify active partition flipped + ansible.builtin.raw: "{{ pathprefix }} fw_printenv active | awk -F= '/^active=/{print $2}'" + register: active_after + changed_when: false + failed_when: (active_after.stdout | trim) != new_active + + - name: Create prepared marker + ansible.builtin.raw: "{{ pathprefix }} touch '{{ fw_marker }}'" + changed_when: true + + - name: Build control queue payload (progress & target version) + ansible.builtin.set_fact: + nbq2_payload_obj: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "update_wo_restart" + task_result: "waiting_restart" + task_add1: "{{ fw_banner_repr }}" # e.g., "2.2.0 rev 9739" + when: up_write is changed + + - name: Publish message to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ nbq2_payload_obj | to_json }}" + payload_encoding: "string" + register: rmq_resp + changed_when: (rmq_resp.json is defined) and (rmq_resp.json.routed | default(false)) + failed_when: > + (rmq_resp.status != 200) or + (rmq_resp.json is not defined) or + (rmq_resp.json.routed | default(false) | bool == false) + delegate_to: localhost + when: nbq2_payload_obj is defined + + - name: Log control queue publish result + ansible.builtin.debug: + var: rmq_resp.json + when: rmq_resp is defined + + - name: Summary + ansible.builtin.debug: + msg: + - "Banner before: {{ banner.stdout | trim }}" + - "Target version: {{ fw_banner_repr }}" + - "SHA256: OK ({{ sha_out.stdout | trim }})" + - "update -c: {{ up_check.stdout | trim }}" + - "update -w: OK" + - "active: {{ active_before.stdout | trim }} -> {{ new_active }}" + - "Marker: {{ fw_marker }}" + + # --- Optional scheduled reboot (must be last device-side command) --- + - name: Compute reboot delay in seconds (if rebootin provided) + ansible.builtin.set_fact: + reboot_seconds: "{{ (rebootin | int) * 3600 }}" + when: + - nbq2_payload_obj is defined + - rebootin is defined + + - name: Schedule delayed reboot on device (HUP-safe) + ansible.builtin.raw: > + {{ pathprefix }} + sh -c 'trap "" HUP; reboot -d {{ reboot_seconds }} >/dev/null 2>&1 &' + changed_when: true + when: + - nbq2_payload_obj is defined + - reboot_seconds is defined + + # --- Success tag selection (ONLY CHANGE) --- + - name: Build control queue payload for update-auto-restarted (rebootin == 0) + ansible.builtin.set_fact: + tag_auto_restarted_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "update-auto-restarted" + when: + - nbq2_payload_obj is defined + - rebootin is defined + - (rebootin | int) == 0 + + - name: Build control queue payload for update-restart-scheduled (rebootin >= 1) + ansible.builtin.set_fact: + tag_restart_scheduled_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "update-restart-scheduled" + when: + - nbq2_payload_obj is defined + - rebootin is defined + - (rebootin | int) >= 1 + + - name: Build control queue payload for update-waits-restart tag (no reboot scheduled) + ansible.builtin.set_fact: + tag_waits_restart_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "update-waits-restart" + when: + - nbq2_payload_obj is defined + - rebootin is not defined + + # --- Publish chosen tag (updated names only) --- + - name: Publish update-waits-restart tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_waits_restart_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_waits_restart_resp + changed_when: (rmq_tag_waits_restart_resp.json is defined) and (rmq_tag_waits_restart_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_waits_restart_resp.status != 200) or + (rmq_tag_waits_restart_resp.json is not defined) or + (not (rmq_tag_waits_restart_resp.json.routed | default(false) | bool)) + delegate_to: localhost + when: tag_waits_restart_payload is defined + + - name: Publish update-auto-restarted tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_auto_restarted_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_auto_restarted_resp + changed_when: (rmq_tag_auto_restarted_resp.json is defined) and (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_auto_restarted_resp.status != 200) or + (rmq_tag_auto_restarted_resp.json is not defined) or + (not (rmq_tag_auto_restarted_resp.json.routed | default(false) | bool)) + delegate_to: localhost + when: tag_auto_restarted_payload is defined + + - name: Publish update-restart-scheduled tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_restart_scheduled_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_restart_scheduled_resp + changed_when: (rmq_tag_restart_scheduled_resp.json is defined) and (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_restart_scheduled_resp.status != 200) or + (rmq_tag_restart_scheduled_resp.json is not defined) or + (not (rmq_tag_restart_scheduled_resp.json.routed | default(false) | bool)) + delegate_to: localhost + when: tag_restart_scheduled_payload is defined + + # --- Journal: preparation successful (only if fully successful) --- + - name: Build control queue payload for success journal + ansible.builtin.set_fact: + journal_success_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + Preparation complete for {{ fw_banner_repr }}. + Active {{ active_before.stdout | trim }} → {{ new_active }}; + marker {{ fw_marker }}. + {{ + ('Scheduled restart in ' ~ (rebootin | int) ~ ' hours to activate new firmware.') + if (rebootin is defined) + else 'Waiting for restart to activate new firmware.' + }} + when: nbq2_payload_obj is defined + + - name: Publish success journal to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_success_payload | to_json }}" + payload_encoding: "string" + register: rmq_journal_success_resp + changed_when: (rmq_journal_success_resp.json is defined) and (rmq_journal_success_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_success_resp.status != 200) or + (rmq_journal_success_resp.json is not defined) or + (not (rmq_journal_success_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + # ----------------- NEW: schedule "afterupgrade_check" message ------------- + # Architectural notes: + # - Only schedule if preparation succeeded (nbq2_payload_obj set) + # - First attempt waits 5 minutes (300s). Retries/backoff are handled by the consumer + # by re-enqueuing fresh messages with increased delays; the producer does NOT sleep. + # - We publish to the holding exchange with AMQP per-message TTL ("expiration" in ms). + # After TTL, the holding queue dead-letters to exchange 'deviceconfig'. + - name: Init after-upgrade scheduling vars + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m). + # Because this task runs with delegate_to: localhost, read from hostvars. + au_delay_sec: >- + {{ + ( + (hostvars[inventory_hostname].reboot_seconds | default(0) | int) + + 300 + ) + if (hostvars[inventory_hostname].reboot_seconds is defined) + else 300 + }} + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Init after-upgrade scheduling vars + ansible.builtin.set_fact: + au_attempt: 1 + au_max_attempts: 3 + # If a reboot was scheduled on the target, wait reboot_seconds + 300s (5m). + # Because this task runs with delegate_to: localhost, read from hostvars. + au_delay_sec: >- + {{ + ( + (hostvars[inventory_hostname].reboot_seconds | default(0) | int) + + 300 + ) + if (hostvars[inventory_hostname].reboot_seconds is defined) + else 300 + }} + when: nbq2_payload_obj is defined + delegate_to: localhost + + # NEW: compute values that the payload will reference + - name: Generate correlation ID and original emitted timestamp + ansible.builtin.set_fact: + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + afterupgrade_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr }}" + attempt: "{{ au_attempt | default(1) }}" + max_attempts: "{{ au_max_attempts | default(3) }}" + current_delay_sec: "{{ au_delay_sec | default(300) }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Build after-upgrade check payload (attempt 1) + ansible.builtin.set_fact: + au_attempt: 1 + au_delay_sec: 300 + au_correlation_id: "{{ lookup('pipe', 'date +%s%N | sha1sum | cut -c1-12') }}" + au_original_emitted_at: "{{ lookup('pipe', 'date -u +%FT%TZ') }}" + afterupgrade_payload: + task_name: "afterupgrade_check" + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + target_version: "{{ fw_banner_repr }}" + attempt: "{{ au_attempt | default(1) }}" + max_attempts: "{{ au_max_attempts | default(3) }}" + current_delay_sec: "{{ au_delay_sec | default(300) }}" + correlation_id: "{{ au_correlation_id }}" + original_emitted_at: "{{ au_original_emitted_at }}" + schema_version: 1 + when: nbq2_payload_obj is defined + delegate_to: localhost + + - name: Debug x-delay about to be sent (ms) + ansible.builtin.debug: + msg: "x-delay(ms) = {{ (au_delay_sec | int) * 1000 }}" + when: afterupgrade_payload is defined + delegate_to: localhost + + - name: Publish delayed after-upgrade check to holding exchange + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ 'deviceconfig.delayed' | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + headers: + x-delay: "{{ (au_delay_sec | int) * 1000 }}" + routing_key: "{{ afterupgrade_routing_key }}" + payload: "{{ afterupgrade_payload | to_json }}" + payload_encoding: "string" + register: rmq_afterupgrade_resp + changed_when: (rmq_afterupgrade_resp.json is defined) and (rmq_afterupgrade_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_afterupgrade_resp.status != 200) or + (rmq_afterupgrade_resp.json is not defined) + when: afterupgrade_payload is defined + delegate_to: localhost + + rescue: + - name: Build failure task name and detail + ansible.builtin.set_fact: + fail_task_name: "{{ ansible_failed_task.name | default('unknown step') }}" + fail_detail_raw: >- + {{ ansible_failed_result.msg + | default(ansible_failed_result.stderr) + | default(ansible_failed_result.stdout) + | default('no additional error output') + | trim }} + + - name: Build failure summary text + ansible.builtin.set_fact: + fail_summary: >- + Firmware update aborted at '{{ fail_task_name }}': {{ fail_detail_raw }} + + - name: Truncate failure summary to ~400 chars + ansible.builtin.set_fact: + fail_summary_short: "{{ fail_summary | regex_replace('\\s+', ' ') | trim | truncate(400, True, '...') }}" + + - name: Build control queue payload for failure journal + ansible.builtin.set_fact: + journal_failure_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: "{{ fail_summary_short }}" + + - name: Publish failure journal to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ journal_failure_payload | to_json }}" + payload_encoding: "string" + register: rmq_journal_fail_resp + changed_when: (rmq_journal_fail_resp.json is defined) and (rmq_journal_fail_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_journal_fail_resp.status != 200) or + (rmq_journal_fail_resp.json is not defined) or + (not (rmq_journal_fail_resp.json.routed | default(false) | bool)) + delegate_to: localhost + + - name: Build control queue payload for update-aborted tag + ansible.builtin.set_fact: + tag_failed_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "tag_add" + task_result: "update-aborted" + + - name: Publish update-aborted tag to control queue via RabbitMQ HTTP API + ansible.builtin.uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ tag_failed_payload | to_json }}" + payload_encoding: "string" + register: rmq_tag_failed_resp + changed_when: (rmq_tag_failed_resp.json is defined) and (rmq_tag_failed_resp.json.routed | default(false) | bool) + failed_when: > + (rmq_tag_failed_resp.status != 200) or + (rmq_tag_failed_resp.json is not defined) or + (not (rmq_tag_failed_resp.json.routed | default(false) | bool)) + delegate_to: localhost diff --git a/files/ansible-playbooks/upgrade-confirmed.yml b/files/ansible-playbooks/upgrade-confirmed.yml new file mode 100644 index 0000000..879d735 --- /dev/null +++ b/files/ansible-playbooks/upgrade-confirmed.yml @@ -0,0 +1,4 @@ +- import_playbook: update-rebootin222.yml + vars: + rebootin: 0 + diff --git a/files/ansible-playbooks/uptime.yml b/files/ansible-playbooks/uptime.yml new file mode 100644 index 0000000..e5897a5 --- /dev/null +++ b/files/ansible-playbooks/uptime.yml @@ -0,0 +1,13 @@ +--- +- name: Run uptime on a host + hosts: all + gather_facts: no + + tasks: + - name: Check uptime + ansible.builtin.raw: uptime + register: uptime_out + + - name: Show result + ansible.builtin.debug: + var: uptime_out.stdout diff --git a/files/ansible-playbooks/wifidebug14.yml b/files/ansible-playbooks/wifidebug14.yml new file mode 100644 index 0000000..92fa8b0 --- /dev/null +++ b/files/ansible-playbooks/wifidebug14.yml @@ -0,0 +1,384 @@ +--- +- name: Deploy WiFi debug (single device, linear) + hosts: all + gather_facts: no + + vars: + remote_syslog_ip: "102.38.125.161" + + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (from env with defaults) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('amq.default', true) }}" + queue2_name: "{{ lookup('env','QUEUE2') | default('queue2', true) }}" + + tasks: + + - block: + + # --- SSH reachability check --- + - name: Check SSH connectivity (raw ping) + raw: "echo ping" + register: ping_result + ignore_errors: true + + - block: + +############ step 1 + + - name: Check if crontab launch is already present in sysstart.lua + raw: | + grep -F 'crontabs -f -l 8' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT + register: sysstart_cron_check + changed_when: false + failed_when: false + + - name: Announce presence and skip edits + when: (sysstart_cron_check.stdout | trim) == 'PRESENT' + debug: + msg: "crontab is present in sysstart.lua; skipping download/edit/upload." + + - name: Fetch sysstart.lua via scp (password auth) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + /tmp/{{ inventory_hostname }}_sysstart.lua + register: scp_get + retries: 3 + delay: 2 + until: scp_get.rc == 0 + + - name: Edit the file locally + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + lineinfile: + path: "/tmp/{{ inventory_hostname }}_sysstart.lua" + line: 'launchd.set_process("crontab", "crond -c /etc/crontabs -f -l 8")' + insertbefore: '^tasks\.start\(ctx,\s*"finished"\)' + + - name: Push the file back (write to -2) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + /tmp/{{ inventory_hostname }}_sysstart.lua + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + register: scp_put + retries: 3 + delay: 2 + until: scp_put.rc == 0 + +############ step 2 + + - name: Compute MD5 of local wifidebug.sh + delegate_to: localhost + command: md5sum files/wifidebug.sh + register: md5_local_wdbg + changed_when: false + + - name: Compute MD5 of remote /root/wifidebug.sh + raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh" + register: md5_remote_wdbg + changed_when: false + failed_when: false + + - name: Decide if wifidebug.sh needs upload + set_fact: + upload_wdbg: >- + {{ (md5_remote_wdbg.rc != 0) + or ((md5_local_wdbg.stdout.split()[0]) + != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }} + + - name: Upload wifidebug.sh via scp (overwrite if changed) + when: upload_wdbg | bool + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/wifidebug.sh + {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh + register: scp_wifidebug + retries: 3 + delay: 2 + until: scp_wifidebug.rc == 0 + + - name: Ensure /root/wifidebug.sh is executable and owned by root + raw: | + chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh + +############ step 3 + + - name: Ensure /etc/crontabs/root exists (touch with perms) + raw: | + if [ ! -f /etc/crontabs/root ]; then + touch /etc/crontabs/root + fi + chown root:root /etc/crontabs/root + chmod 0644 /etc/crontabs/root + + - name: Check if cron line already present + raw: | + grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root + register: cron_grep + failed_when: false + changed_when: false + + - name: Upload snippet crond-root to /tmp (only if missing) + when: cron_grep.rc != 0 + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/crond-root + {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet + + - name: Append snippet to /etc/crontabs/root (only if missing) + when: cron_grep.rc != 0 + raw: | + cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet + register: cron_append + changed_when: true + +############ step 4 + + - name: Skip config.json edit if remote_syslog already matches + raw: | + EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0) + SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0) + if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then + echo "SKIP" + else + echo "EDIT" + fi + register: remote_syslog_check + changed_when: false + + - name: Set do_edit flag from skip check + set_fact: + do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}" + + - name: Fetch /tmp/config.json to controller + when: do_edit + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json + /tmp/{{ inventory_hostname }}_config.json + + - name: Strict in-place style-preserving edit (services.remote_syslog only) + when: do_edit + delegate_to: localhost + shell: | + awk ' + BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 } + { + line = $0 + if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 } + if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 } + if (in_rs) { + if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line) + done_enabled=1 + } + if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line) + done_server=1 + } + } + print line + if (in_rs) { + opens = gsub(/{/, "{", $0) + closes = gsub(/}/, "}", $0) + depth += (opens - closes) + if (depth <= 0 && $0 ~ /}/) { in_rs=0 } + } + } + ' "/tmp/{{ inventory_hostname }}_config.json" \ + > "/tmp/{{ inventory_hostname }}_config.json.new" + args: + executable: /bin/bash + + - name: Push edited config.json.new to remote temp + when: do_edit + delegate_to: localhost + shell: | + sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \ + "/tmp/{{ inventory_hostname }}_config.json.new" \ + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new + args: + executable: /bin/bash + + - name: Compute MD5 of local .new + when: do_edit + delegate_to: localhost + command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new" + register: md5_local + changed_when: false + + - name: Compute MD5 of remote .new (no Python) + when: do_edit + raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new" + register: md5_remote + changed_when: false + + - name: Fail if MD5 mismatch + when: + - do_edit + - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD') + fail: + msg: "MD5 mismatch between controller and remote copy — aborting replace." + + - name: Commit new config.json (overwrite original) + when: do_edit + raw: | + cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S) + mv /tmp/config.json.new /tmp/config.json + + - name: Apply config to runtime (sysconf -w) + when: do_edit + raw: | + sysconf -w || /usr/sbin/sysconf -w + register: sysconf_result + changed_when: true + failed_when: sysconf_result.rc not in [0] + + - name: Trigger full restart (system-stop; system-start) in background + when: do_edit + raw: | + sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start + ignore_errors: true + changed_when: true + + - name: Short grace delay before waiting + when: do_edit + delegate_to: localhost + pause: + seconds: 3 + + - name: Try to observe SSH port stopping (best effort) + when: do_edit + delegate_to: localhost + wait_for: + host: "{{ ansible_host }}" + port: 22 + state: stopped + timeout: 30 + sleep: 2 + ignore_errors: true + + - name: Wait for SSH port to be accepting connections + when: do_edit + delegate_to: localhost + wait_for: + host: "{{ ansible_host }}" + port: 22 + state: started + timeout: 300 + sleep: 2 + + - name: Reset Ansible SSH connection + when: do_edit + meta: reset_connection + + - name: Verify command execution after restart (no python) + when: do_edit + raw: "echo rebooted_ok" + register: post_restart_probe + retries: 120 + delay: 2 + until: post_restart_probe is succeeded + changed_when: false + + - name: Set result status (success deployed or no change) + set_fact: + result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}" + + when: ping_result is succeeded + + - name: Set status fact (no ssh) + when: ping_result is failed + set_fact: + result_status: "NO_SSH" + + rescue: + - name: Mark result as failed + set_fact: + result_status: "FAILED during {{ ansible_failed_task.name }}" + + always: + + - name: Compute inscope device + set_fact: + inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}" + + - name: Build journal_add payload + set_fact: + journal_payload_obj: + inscope_device: "{{ inscope_device_name }}" + task_name: "journal_add" + task_result: "{{ result_status }}" + + - name: Publish journal_add to queue2 + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ queue2_name }}" + payload: "{{ journal_payload_obj | to_json }}" + payload_encoding: "string" + delegate_to: localhost + + - name: Build deploy_wifidebug success payload + when: result_status == 'SUCCESS_DEPLOYED' + set_fact: + success_payload_obj: + inscope_device: "{{ inscope_device_name }}" + task_name: "deploy_wifidebug" + task_result: "success" + + - name: Publish deploy_wifidebug success to queue2 + when: result_status == 'SUCCESS_DEPLOYED' + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ queue2_name }}" + payload: "{{ success_payload_obj | to_json }}" + payload_encoding: "string" + delegate_to: localhost + + - name: Summary + debug: + msg: + - "result_status: {{ result_status }}" + - "we're good" + diff --git a/files/ansible-playbooks/wifidebug15.yml b/files/ansible-playbooks/wifidebug15.yml new file mode 100644 index 0000000..0b589cc --- /dev/null +++ b/files/ansible-playbooks/wifidebug15.yml @@ -0,0 +1,447 @@ +--- +- name: Deploy WiFi debug (single device, linear) + hosts: all + gather_facts: no + + vars: + remote_syslog_ip: "102.38.125.161" + + # Program version (0) + wdbg_version: "v15" + + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (use controls exchange + queue_controls like the reference) (e) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + tasks: + + - block: + + # --- SSH reachability check --- + - name: Check SSH connectivity (raw ping) + raw: "echo ping" + register: ping_result + ignore_errors: true + + - block: + +############ step 1 + + - name: Check if crontab launch is already present in sysstart.lua + raw: | + grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT + register: sysstart_cron_check + changed_when: false + failed_when: false + + - name: Announce presence and skip edits + when: (sysstart_cron_check.stdout | trim) == 'PRESENT' + debug: + msg: "crontab is present in sysstart.lua; skipping download/edit/upload." + + - name: Fetch sysstart.lua via scp (password auth) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + /tmp/{{ inventory_hostname }}_sysstart.lua + register: scp_get + retries: 3 + delay: 2 + until: scp_get.rc == 0 + + - name: Edit the file locally + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + lineinfile: + path: "/tmp/{{ inventory_hostname }}_sysstart.lua" + line: 'launchd.set_process("crond", "/usr/sbin/crond -f")' + insertbefore: '^tasks\.start\(ctx,\s*"finished"\)' + + - name: Push the file back (write to -2) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + /tmp/{{ inventory_hostname }}_sysstart.lua + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + register: scp_put + retries: 3 + delay: 2 + until: scp_put.rc == 0 + +############ step 2 + + - name: Compute MD5 of local wifidebug.sh + delegate_to: localhost + command: md5sum files/wifidebug.sh + register: md5_local_wdbg + changed_when: false + + - name: Compute MD5 of remote /root/wifidebug.sh + raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh" + register: md5_remote_wdbg + changed_when: false + failed_when: false + + - name: Decide if wifidebug.sh needs upload + set_fact: + upload_wdbg: >- + {{ (md5_remote_wdbg.rc != 0) + or ((md5_local_wdbg.stdout.split()[0]) + != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }} + + - name: Upload wifidebug.sh via scp (overwrite if changed) + when: upload_wdbg | bool + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/wifidebug.sh + {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh + register: scp_wifidebug + retries: 3 + delay: 2 + until: scp_wifidebug.rc == 0 + + - name: Ensure /root/wifidebug.sh is executable and owned by root + raw: | + chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh + +############ step 3 + + - name: Ensure /etc/crontabs/root exists (touch with perms) + raw: | + if [ ! -f /etc/crontabs/root ]; then + touch /etc/crontabs/root + fi + chown root:root /etc/crontabs/root + chmod 0644 /etc/crontabs/root + + - name: Check if cron line already present + raw: | + grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root + register: cron_grep + failed_when: false + changed_when: false + + - name: Upload snippet crond-root to /tmp (only if missing) + when: cron_grep.rc != 0 + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/crond-root + {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet + + - name: Append snippet to /etc/crontabs/root (only if missing) + when: cron_grep.rc != 0 + raw: | + cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet + register: cron_append + changed_when: true + +############ step 4 + + - name: Skip config.json edit if remote_syslog already matches + raw: | + EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0) + SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0) + if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then + echo "SKIP" + else + echo "EDIT" + fi + register: remote_syslog_check + changed_when: false + + - name: Set do_edit flag from skip check + set_fact: + do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}" + + - name: Fetch /tmp/config.json to controller + when: do_edit + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json + /tmp/{{ inventory_hostname }}_config.json + + - name: Strict in-place style-preserving edit (services.remote_syslog only) + when: do_edit + delegate_to: localhost + shell: | + awk ' + BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 } + { + line = $0 + if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 } + if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 } + if (in_rs) { + if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line) + done_enabled=1 + } + if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line) + done_server=1 + } + } + print line + if (in_rs) { + opens = gsub(/{/, "{", $0) + closes = gsub(/}/, "}", $0) + depth += (opens - closes) + if (depth <= 0 && $0 ~ /}/) { in_rs=0 } + } + } + ' "/tmp/{{ inventory_hostname }}_config.json" \ + > "/tmp/{{ inventory_hostname }}_config.json.new" + args: + executable: /bin/bash + + - name: Push edited config.json.new to remote temp + when: do_edit + delegate_to: localhost + shell: | + sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \ + "/tmp/{{ inventory_hostname }}_config.json.new" \ + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new + args: + executable: /bin/bash + + - name: Compute MD5 of local .new + when: do_edit + delegate_to: localhost + command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new" + register: md5_local + changed_when: false + + - name: Compute MD5 of remote .new (no Python) + when: do_edit + raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new" + register: md5_remote + changed_when: false + + - name: Fail if MD5 mismatch + when: + - do_edit + - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD') + fail: + msg: "MD5 mismatch between controller and remote copy — aborting replace." + + - name: Commit new config.json (overwrite original) + when: do_edit + raw: | + cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S) + mv /tmp/config.json.new /tmp/config.json + + - name: Apply config to runtime (sysconf -w) + when: do_edit + raw: | + sysconf -w || /usr/sbin/sysconf -w + register: sysconf_result + changed_when: true + failed_when: sysconf_result.rc not in [0] + + # (d) Pre-restart notice to control queue + - name: Build pre-restart journal (explain why) + when: do_edit + delegate_to: localhost + set_fact: + prereboot_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w). + + - name: Publish pre-restart journal to control queue + when: do_edit + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ prereboot_payload | to_json }}" + payload_encoding: "string" + register: rmq_preboot + changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool) + + - name: Trigger full restart (system-stop; system-start) in background + when: do_edit + raw: | + sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start + ignore_errors: true + changed_when: true + + - name: Short grace delay before waiting + when: do_edit + delegate_to: localhost + pause: + seconds: 3 + + - name: Try to observe SSH port stopping (best effort) + when: do_edit + delegate_to: localhost + wait_for: + host: "{{ ansible_host }}" + port: 22 + state: stopped + timeout: 30 + sleep: 2 + ignore_errors: true + + - name: Wait for SSH port to be accepting connections + when: do_edit + delegate_to: localhost + wait_for: + host: "{{ ansible_host }}" + port: 22 + state: started + timeout: 300 + sleep: 2 + + - name: Reset Ansible SSH connection + when: do_edit + meta: reset_connection + + - name: Verify command execution after restart (no python) + when: do_edit + raw: "echo rebooted_ok" + register: post_restart_probe + retries: 120 + delay: 2 + until: post_restart_probe is succeeded + changed_when: false + + - name: Set result status (success deployed or no change) + set_fact: + result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}" + + when: ping_result is succeeded + + - name: Set status fact (no ssh) + when: ping_result is failed + set_fact: + result_status: "NO_SSH" + + rescue: + - name: Mark result as failed + set_fact: + result_status: "FAILED during {{ ansible_failed_task.name }}" + + always: + + - name: Compute inscope device + set_fact: + inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}" + + # (a) Set custom field wifidebug -> v15 on control queue + + - name: Build custom-field payload (wifidebug -> version) + delegate_to: localhost + set_fact: + wdbg_cf_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "deploy_wifidebug" + task_result: "{{ wdbg_version }}" + + - name: Publish custom-field update to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wdbg_cf_payload | to_json }}" + payload_encoding: "string" + register: rmq_cf + changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool) + + # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed + + - name: Build actions list + set_fact: + _actions_list: >- + {{ + [] + + ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], [])) + + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], [])) + + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], [])) + + ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], [])) + + ((do_edit | default(false) | bool) | ternary(['restarted services'], [])) + }} + + - name: Build actions string + set_fact: + _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}" + + + - name: Build wrap-up journal payload + delegate_to: localhost + set_fact: + wrap_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "journal_add" + task_result: >- + wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }} + — actions: {{ _actions_str }} + + - name: Publish wrap-up journal to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wrap_payload | to_json }}" + payload_encoding: "string" + register: rmq_wrap + changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool) + + # Local summary (kept for operator visibility) + - name: Summary + debug: + msg: + - "result_status: {{ result_status }}" + - "we're good" diff --git a/files/ansible-playbooks/wifidebug16.yml b/files/ansible-playbooks/wifidebug16.yml new file mode 100644 index 0000000..72c0d77 --- /dev/null +++ b/files/ansible-playbooks/wifidebug16.yml @@ -0,0 +1,447 @@ +--- +- name: Deploy WiFi debug (single device, linear) + hosts: all + gather_facts: no + + vars: + remote_syslog_ip: "102.38.125.161" + + # Program version (0) + wdbg_version: "v16" + + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (use controls exchange + queue_controls like the reference) (e) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + tasks: + + - block: + + # --- SSH reachability check --- + - name: Check SSH connectivity (raw ping) + raw: "echo ping" + register: ping_result + ignore_errors: true + + - block: + +############ step 1 + + - name: Check if crontab launch is already present in sysstart.lua + raw: | + grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT + register: sysstart_cron_check + changed_when: false + failed_when: false + + - name: Announce presence and skip edits + when: (sysstart_cron_check.stdout | trim) == 'PRESENT' + debug: + msg: "crontab is present in sysstart.lua; skipping download/edit/upload." + + - name: Fetch sysstart.lua via scp (password auth) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + /tmp/{{ inventory_hostname }}_sysstart.lua + register: scp_get + retries: 3 + delay: 2 + until: scp_get.rc == 0 + + - name: Edit the file locally + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + lineinfile: + path: "/tmp/{{ inventory_hostname }}_sysstart.lua" + line: 'launchd.set_process("crond", "/usr/sbin/crond -f")' + insertbefore: '^tasks\.start\(ctx,\s*"finished"\)' + + - name: Push the file back (write to -2) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + /tmp/{{ inventory_hostname }}_sysstart.lua + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + register: scp_put + retries: 3 + delay: 2 + until: scp_put.rc == 0 + +############ step 2 + + - name: Compute MD5 of local wifidebug.sh + delegate_to: localhost + command: md5sum files/wifidebug.sh + register: md5_local_wdbg + changed_when: false + + - name: Compute MD5 of remote /root/wifidebug.sh + raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh" + register: md5_remote_wdbg + changed_when: false + failed_when: false + + - name: Decide if wifidebug.sh needs upload + set_fact: + upload_wdbg: >- + {{ (md5_remote_wdbg.rc != 0) + or ((md5_local_wdbg.stdout.split()[0]) + != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }} + + - name: Upload wifidebug.sh via scp (overwrite if changed) + when: upload_wdbg | bool + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/wifidebug.sh + {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh + register: scp_wifidebug + retries: 3 + delay: 2 + until: scp_wifidebug.rc == 0 + + - name: Ensure /root/wifidebug.sh is executable and owned by root + raw: | + chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh + +############ step 3 + + - name: Ensure /etc/crontabs/root exists (touch with perms) + raw: | + if [ ! -f /etc/crontabs/root ]; then + touch /etc/crontabs/root + fi + chown root:root /etc/crontabs/root + chmod 0644 /etc/crontabs/root + + - name: Check if cron line already present + raw: | + grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root + register: cron_grep + failed_when: false + changed_when: false + + - name: Upload snippet crond-root to /tmp (only if missing) + when: cron_grep.rc != 0 + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/crond-root + {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet + + - name: Append snippet to /etc/crontabs/root (only if missing) + when: cron_grep.rc != 0 + raw: | + cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet + register: cron_append + changed_when: true + +############ step 4 + + - name: Skip config.json edit if remote_syslog already matches + raw: | + EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0) + SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0) + if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then + echo "SKIP" + else + echo "EDIT" + fi + register: remote_syslog_check + changed_when: false + + - name: Set do_edit flag from skip check + set_fact: + do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}" + + - name: Fetch /tmp/config.json to controller + when: do_edit + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json + /tmp/{{ inventory_hostname }}_config.json + + - name: Strict in-place style-preserving edit (services.remote_syslog only) + when: do_edit + delegate_to: localhost + shell: | + awk ' + BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 } + { + line = $0 + if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 } + if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 } + if (in_rs) { + if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line) + done_enabled=1 + } + if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line) + done_server=1 + } + } + print line + if (in_rs) { + opens = gsub(/{/, "{", $0) + closes = gsub(/}/, "}", $0) + depth += (opens - closes) + if (depth <= 0 && $0 ~ /}/) { in_rs=0 } + } + } + ' "/tmp/{{ inventory_hostname }}_config.json" \ + > "/tmp/{{ inventory_hostname }}_config.json.new" + args: + executable: /bin/bash + + - name: Push edited config.json.new to remote temp + when: do_edit + delegate_to: localhost + shell: | + sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \ + "/tmp/{{ inventory_hostname }}_config.json.new" \ + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new + args: + executable: /bin/bash + + - name: Compute MD5 of local .new + when: do_edit + delegate_to: localhost + command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new" + register: md5_local + changed_when: false + + - name: Compute MD5 of remote .new (no Python) + when: do_edit + raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new" + register: md5_remote + changed_when: false + + - name: Fail if MD5 mismatch + when: + - do_edit + - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD') + fail: + msg: "MD5 mismatch between controller and remote copy — aborting replace." + + - name: Commit new config.json (overwrite original) + when: do_edit + raw: | + cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S) + mv /tmp/config.json.new /tmp/config.json + + - name: Apply config to runtime (sysconf -w) + when: do_edit + raw: | + sysconf -w || /usr/sbin/sysconf -w + register: sysconf_result + changed_when: true + failed_when: sysconf_result.rc not in [0] + + # (d) Pre-restart notice to control queue + - name: Build pre-restart journal (explain why) + when: do_edit + delegate_to: localhost + set_fact: + prereboot_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w). + + - name: Publish pre-restart journal to control queue + when: do_edit + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ prereboot_payload | to_json }}" + payload_encoding: "string" + register: rmq_preboot + changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool) + + - name: Trigger full restart (system-stop; system-start) in background + when: do_edit + raw: | + sleep 1; /usr/sbin/system-stop; /usr/sbin/system-start + ignore_errors: true + changed_when: true + + - name: Short grace delay before waiting + when: do_edit + delegate_to: localhost + pause: + seconds: 3 + + - name: Try to observe SSH port stopping (best effort) + when: do_edit + delegate_to: localhost + wait_for: + host: "{{ ansible_host }}" + port: 22 + state: stopped + timeout: 30 + sleep: 2 + ignore_errors: true + + - name: Wait for SSH port to be accepting connections + when: do_edit + delegate_to: localhost + wait_for: + host: "{{ ansible_host }}" + port: 22 + state: started + timeout: 300 + sleep: 2 + + - name: Reset Ansible SSH connection + when: do_edit + meta: reset_connection + + - name: Verify command execution after restart (no python) + when: do_edit + raw: "echo rebooted_ok" + register: post_restart_probe + retries: 120 + delay: 2 + until: post_restart_probe is succeeded + changed_when: false + + - name: Set result status (success deployed or no change) + set_fact: + result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}" + + when: ping_result is succeeded + + - name: Set status fact (no ssh) + when: ping_result is failed + set_fact: + result_status: "NO_SSH" + + rescue: + - name: Mark result as failed + set_fact: + result_status: "FAILED during {{ ansible_failed_task.name }}" + + always: + + - name: Compute inscope device + set_fact: + inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}" + + # (a) Set custom field wifidebug -> v15 on control queue + + - name: Build custom-field payload (wifidebug -> version) + delegate_to: localhost + set_fact: + wdbg_cf_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "deploy_wifidebug" + task_result: "{{ wdbg_version }}" + + - name: Publish custom-field update to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wdbg_cf_payload | to_json }}" + payload_encoding: "string" + register: rmq_cf + changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool) + + # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed + + - name: Build actions list + set_fact: + _actions_list: >- + {{ + [] + + ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], [])) + + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], [])) + + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], [])) + + ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], [])) + + ((do_edit | default(false) | bool) | ternary(['restarted services'], [])) + }} + + - name: Build actions string + set_fact: + _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}" + + + - name: Build wrap-up journal payload + delegate_to: localhost + set_fact: + wrap_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "journal_add" + task_result: >- + wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }} + — actions: {{ _actions_str }} + + - name: Publish wrap-up journal to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wrap_payload | to_json }}" + payload_encoding: "string" + register: rmq_wrap + changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool) + + # Local summary (kept for operator visibility) + - name: Summary + debug: + msg: + - "result_status: {{ result_status }}" + - "we're good" diff --git a/files/ansible-playbooks/wifidebug17.yml b/files/ansible-playbooks/wifidebug17.yml new file mode 100644 index 0000000..0795173 --- /dev/null +++ b/files/ansible-playbooks/wifidebug17.yml @@ -0,0 +1,476 @@ +--- +- name: Deploy WiFi debug (single device, linear) + hosts: all + gather_facts: no + + vars: + remote_syslog_ip: "102.38.125.161" + + # Program version (0) + wdbg_version: "v17" + + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (use controls exchange + queue_controls like the reference) (e) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + tasks: + + - block: + + # --- SSH reachability check --- + - name: Check SSH connectivity (raw ping) + raw: "echo ping" + register: ping_result + ignore_errors: true + + - block: + +############ step 1 + + - name: Check if crontab launch is already present in sysstart.lua + raw: | + grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT + register: sysstart_cron_check + changed_when: false + failed_when: false + + - name: Announce presence and skip edits + when: (sysstart_cron_check.stdout | trim) == 'PRESENT' + debug: + msg: "crontab is present in sysstart.lua; skipping download/edit/upload." + + - name: Fetch sysstart.lua via scp (password auth) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + /tmp/{{ inventory_hostname }}_sysstart.lua + register: scp_get + retries: 3 + delay: 2 + until: scp_get.rc == 0 + + - name: Edit the file locally + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + lineinfile: + path: "/tmp/{{ inventory_hostname }}_sysstart.lua" + line: 'launchd.set_process("crond", "/usr/sbin/crond -f")' + insertbefore: '^tasks\.start\(ctx,\s*"finished"\)' + + - name: Push the file back (write to -2) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + /tmp/{{ inventory_hostname }}_sysstart.lua + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + register: scp_put + retries: 3 + delay: 2 + until: scp_put.rc == 0 + +############ step 2 + + - name: Compute MD5 of local wifidebug.sh + delegate_to: localhost + command: md5sum files/wifidebug.sh + register: md5_local_wdbg + changed_when: false + + - name: Compute MD5 of remote /root/wifidebug.sh + raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh" + register: md5_remote_wdbg + changed_when: false + failed_when: false + + - name: Decide if wifidebug.sh needs upload + set_fact: + upload_wdbg: >- + {{ (md5_remote_wdbg.rc != 0) + or ((md5_local_wdbg.stdout.split()[0]) + != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }} + + - name: Upload wifidebug.sh via scp (overwrite if changed) + when: upload_wdbg | bool + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/wifidebug.sh + {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh + register: scp_wifidebug + retries: 3 + delay: 2 + until: scp_wifidebug.rc == 0 + + - name: Ensure /root/wifidebug.sh is executable and owned by root + raw: | + chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh + +############ step 3 + + - name: Ensure /etc/crontabs/root exists (touch with perms) + raw: | + if [ ! -f /etc/crontabs/root ]; then + touch /etc/crontabs/root + fi + chown root:root /etc/crontabs/root + chmod 0644 /etc/crontabs/root + + - name: Check if cron line already present + raw: | + grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root + register: cron_grep + failed_when: false + changed_when: false + + - name: Upload snippet crond-root to /tmp (only if missing) + when: cron_grep.rc != 0 + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/crond-root + {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet + + - name: Append snippet to /etc/crontabs/root (only if missing) + when: cron_grep.rc != 0 + raw: | + cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet + register: cron_append + changed_when: true + +############ step 4 + + - name: Skip config.json edit if remote_syslog already matches + raw: | + EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0) + SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0) + if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then + echo "SKIP" + else + echo "EDIT" + fi + register: remote_syslog_check + changed_when: false + + - name: Set do_edit flag from skip check + set_fact: + do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}" + + # ===== ADDED: pre-download MD5 on device to guard download integrity ===== + - name: MD5 of remote /tmp/config.json (pre-download) + when: do_edit + raw: "md5sum /tmp/config.json || busybox md5sum /tmp/config.json" + register: md5_remote_src + changed_when: false + # ===== END ADDED ===== + + - name: Fetch /tmp/config.json to controller + when: do_edit + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json + /tmp/{{ inventory_hostname }}_config.json + + # ===== ADDED: verify download integrity by comparing MD5s ===== + - name: MD5 of local downloaded config.json + when: do_edit + delegate_to: localhost + command: md5sum "/tmp/{{ inventory_hostname }}_config.json" + register: md5_local_src + changed_when: false + + - name: Fail if MD5 mismatch on downloaded config.json + when: + - do_edit + - (md5_local_src.stdout.split()[0]) != (md5_remote_src.stdout.split()[0] if (md5_remote_src.stdout is defined) else 'BAD') + fail: + msg: "MD5 mismatch on downloaded /tmp/config.json — aborting (download integrity check failed)." + # ===== END ADDED ===== + + - name: Strict in-place style-preserving edit (services.remote_syslog only) + when: do_edit + delegate_to: localhost + shell: | + awk ' + BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 } + { + line = $0 + if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 } + if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 } + if (in_rs) { + if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line) + done_enabled=1 + } + if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line) + done_server=1 + } + } + print line + if (in_rs) { + opens = gsub(/{/, "{", $0) + closes = gsub(/}/, "}", $0) + depth += (opens - closes) + if (depth <= 0 && $0 ~ /}/) { in_rs=0 } + } + } + ' "/tmp/{{ inventory_hostname }}_config.json" \ + > "/tmp/{{ inventory_hostname }}_config.json.new" + args: + executable: /bin/bash + + # ===== ADDED: controller-side JSON sanity check (jq) ===== + - name: Controller JSON sanity check (jq) + when: do_edit + delegate_to: localhost + shell: jq empty "/tmp/{{ inventory_hostname }}_config.json.new" + args: + executable: /bin/bash + register: jq_local + changed_when: false + failed_when: jq_local.rc != 0 + # ===== END ADDED ===== + + - name: Push edited config.json.new to remote temp + when: do_edit + delegate_to: localhost + shell: | + sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \ + "/tmp/{{ inventory_hostname }}_config.json.new" \ + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new + args: + executable: /bin/bash + + - name: Compute MD5 of local .new + when: do_edit + delegate_to: localhost + command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new" + register: md5_local + changed_when: false + + - name: Compute MD5 of remote .new (no Python) + when: do_edit + raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new" + register: md5_remote + changed_when: false + + - name: Fail if MD5 mismatch + when: + - do_edit + - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD') + fail: + msg: "MD5 mismatch between controller and remote copy — aborting replace." + + - name: Commit new config.json (overwrite original) + when: do_edit + raw: | + cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S) + mv /tmp/config.json.new /tmp/config.json + + - name: Apply config to runtime (sysconf -w) + when: do_edit + raw: | + sysconf -w || /usr/sbin/sysconf -w + register: sysconf_result + changed_when: true + failed_when: sysconf_result.rc not in [0] + + # (d) Pre-restart notice to control queue + - name: Build pre-restart journal (explain why) + when: do_edit + delegate_to: localhost + set_fact: + prereboot_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w). + + - name: Publish pre-restart journal to control queue + when: do_edit + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ prereboot_payload | to_json }}" + payload_encoding: "string" + register: rmq_preboot + changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool) + + # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ======== + + - name: Trigger full restart (system-stop; system-start) from controller with 10s cap + when: do_edit + delegate_to: localhost + shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'" + args: + executable: /bin/bash + register: restart_kick + changed_when: true + failed_when: false + + + # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3) + - name: Probe SSH with nc (24 tries, 5s each) + when: do_edit + delegate_to: localhost + shell: "nc -z -w 3 {{ ansible_host }} 22" + register: nc_probe + retries: 24 + delay: 5 + until: nc_probe.rc == 0 + changed_when: false + failed_when: false + + - name: Set ssh_up fact (from nc probe) + when: do_edit + delegate_to: localhost + set_fact: + ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}" + + - name: Fail if SSH did not return after restart + when: + - do_edit + - not ssh_up | bool + fail: + msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks." + + # ======== END OF SURGICAL CHANGE ======== + + - name: Set result status (success deployed or no change) + set_fact: + result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}" + + when: ping_result is succeeded + + - name: Set status fact (no ssh) + when: ping_result is failed + set_fact: + result_status: "NO_SSH" + + rescue: + - name: Mark result as failed + set_fact: + result_status: "FAILED during {{ ansible_failed_task.name }}" + + always: + + - name: Compute inscope device + set_fact: + inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}" + + # (a) Set custom field wifidebug -> v15 on control queue + + - name: Build custom-field payload (wifidebug -> version) + delegate_to: localhost + set_fact: + wdbg_cf_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "deploy_wifidebug" + task_result: "{{ wdbg_version }}" + + - name: Publish custom-field update to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wdbg_cf_payload | to_json }}" + payload_encoding: "string" + register: rmq_cf + changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool) + + # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed + + - name: Build actions list + set_fact: + _actions_list: >- + {{ + [] + + ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], [])) + + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], [])) + + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], [])) + + ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], [])) + + ((do_edit | default(false) | bool) | ternary(['restarted services'], [])) + }} + + - name: Build actions string + set_fact: + _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}" + + + - name: Build wrap-up journal payload + delegate_to: localhost + set_fact: + wrap_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "journal_add" + task_result: >- + wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }} + — actions: {{ _actions_str }} + + - name: Publish wrap-up journal to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wrap_payload | to_json }}" + payload_encoding: "string" + register: rmq_wrap + changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool) + + # Local summary (kept for operator visibility) + - name: Summary + debug: + msg: + - "result_status: {{ result_status }}" + - "we're good" diff --git a/files/ansible-playbooks/wifidebug17.yml-old b/files/ansible-playbooks/wifidebug17.yml-old new file mode 100644 index 0000000..4184be4 --- /dev/null +++ b/files/ansible-playbooks/wifidebug17.yml-old @@ -0,0 +1,441 @@ +--- +- name: Deploy WiFi debug (single device, linear) + hosts: all + gather_facts: no + + vars: + remote_syslog_ip: "102.38.125.161" + + # Program version (0) + wdbg_version: "v17" + + ssh_user: "{{ ansible_user | default('root') }}" + ssh_pass: "{{ ansible_password | default(ansible_ssh_pass) }}" + + # RabbitMQ (use controls exchange + queue_controls like the reference) (e) + rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}" + rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}" + rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}" + rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}" + rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}" + rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}" + control_queue: "{{ lookup('env','CONTROL_QUEUE') | default('queue_controls', true) }}" + + tasks: + + - block: + + # --- SSH reachability check --- + - name: Check SSH connectivity (raw ping) + raw: "echo ping" + register: ping_result + ignore_errors: true + + - block: + +############ step 1 + + - name: Check if crontab launch is already present in sysstart.lua + raw: | + grep -F 'crond -f' /usr/share/config/sysstart.lua >/dev/null 2>&1 && echo PRESENT || echo ABSENT + register: sysstart_cron_check + changed_when: false + failed_when: false + + - name: Announce presence and skip edits + when: (sysstart_cron_check.stdout | trim) == 'PRESENT' + debug: + msg: "crontab is present in sysstart.lua; skipping download/edit/upload." + + - name: Fetch sysstart.lua via scp (password auth) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + /tmp/{{ inventory_hostname }}_sysstart.lua + register: scp_get + retries: 3 + delay: 2 + until: scp_get.rc == 0 + + - name: Edit the file locally + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + lineinfile: + path: "/tmp/{{ inventory_hostname }}_sysstart.lua" + line: 'launchd.set_process("crond", "/usr/sbin/crond -f")' + insertbefore: '^tasks\.start\(ctx,\s*"finished"\)' + + - name: Push the file back (write to -2) + when: (sysstart_cron_check.stdout | trim) != 'PRESENT' + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + /tmp/{{ inventory_hostname }}_sysstart.lua + {{ ssh_user }}@{{ ansible_host }}:/usr/share/config/sysstart.lua + register: scp_put + retries: 3 + delay: 2 + until: scp_put.rc == 0 + +############ step 2 + + - name: Compute MD5 of local wifidebug.sh + delegate_to: localhost + command: md5sum files/wifidebug.sh + register: md5_local_wdbg + changed_when: false + + - name: Compute MD5 of remote /root/wifidebug.sh + raw: "md5sum /root/wifidebug.sh || busybox md5sum /root/wifidebug.sh" + register: md5_remote_wdbg + changed_when: false + failed_when: false + + - name: Decide if wifidebug.sh needs upload + set_fact: + upload_wdbg: >- + {{ (md5_remote_wdbg.rc != 0) + or ((md5_local_wdbg.stdout.split()[0]) + != (md5_remote_wdbg.stdout.split()[0] if (md5_remote_wdbg.stdout is defined) else '')) }} + + - name: Upload wifidebug.sh via scp (overwrite if changed) + when: upload_wdbg | bool + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/wifidebug.sh + {{ ssh_user }}@{{ ansible_host }}:/root/wifidebug.sh + register: scp_wifidebug + retries: 3 + delay: 2 + until: scp_wifidebug.rc == 0 + + - name: Ensure /root/wifidebug.sh is executable and owned by root + raw: | + chown root:root /root/wifidebug.sh && chmod 0755 /root/wifidebug.sh + +############ step 3 + + - name: Ensure /etc/crontabs/root exists (touch with perms) + raw: | + if [ ! -f /etc/crontabs/root ]; then + touch /etc/crontabs/root + fi + chown root:root /etc/crontabs/root + chmod 0644 /etc/crontabs/root + + - name: Check if cron line already present + raw: | + grep -Fxq '*/1 * * * * /root/wifidebug.sh' /etc/crontabs/root + register: cron_grep + failed_when: false + changed_when: false + + - name: Upload snippet crond-root to /tmp (only if missing) + when: cron_grep.rc != 0 + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + files/crond-root + {{ ssh_user }}@{{ ansible_host }}:/tmp/crond-root.snippet + + - name: Append snippet to /etc/crontabs/root (only if missing) + when: cron_grep.rc != 0 + raw: | + cat /tmp/crond-root.snippet >> /etc/crontabs/root && rm -f /tmp/crond-root.snippet + register: cron_append + changed_when: true + +############ step 4 + + - name: Skip config.json edit if remote_syslog already matches + raw: | + EN_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'enabled' | grep true | wc -l || echo 0) + SRV_OK=$(cat /tmp/config.json | grep -A9 -m1 'remote_syslog' | grep 'server' | grep '{{ remote_syslog_ip }}' | wc -l || echo 0) + if [ "$EN_OK" -eq 1 ] && [ "$SRV_OK" -eq 1 ]; then + echo "SKIP" + else + echo "EDIT" + fi + register: remote_syslog_check + changed_when: false + + - name: Set do_edit flag from skip check + set_fact: + do_edit: "{{ (remote_syslog_check.stdout | default('EDIT')) | trim != 'SKIP' }}" + + - name: Fetch /tmp/config.json to controller + when: do_edit + delegate_to: localhost + command: > + sshpass -p {{ ssh_pass | quote }} + scp -q -o StrictHostKeyChecking=no + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json + /tmp/{{ inventory_hostname }}_config.json + + - name: Strict in-place style-preserving edit (services.remote_syslog only) + when: do_edit + delegate_to: localhost + shell: | + awk ' + BEGIN { in_services=0; in_rs=0; depth=0; done_enabled=0; done_server=0 } + { + line = $0 + if (!in_services && $0 ~ /"services"[[:space:]]*:/) { in_services=1 } + if (in_services && !in_rs && $0 ~ /"remote_syslog"[[:space:]]*:/) { in_rs=1; depth=0 } + if (in_rs) { + if (!done_enabled && line ~ /^[[:space:]]*"enabled"[[:space:]]*:[[:space:]]*false([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"enabled"[[:space:]]*:[[:space:]]*)false([[:space:]]*,?[[:space:]]*)$/, "\\1true\\2", 1, line) + done_enabled=1 + } + if (!done_server && line ~ /^[[:space:]]*"server"[[:space:]]*:[[:space:]]*""([[:space:]]*,?[[:space:]]*)$/) { + line = gensub(/^([[:space:]]*"server"[[:space:]]*:[[:space:]]*)""([[:space:]]*,?[[:space:]]*)$/, "\\1\"102.38.125.161\"\\2", 1, line) + done_server=1 + } + } + print line + if (in_rs) { + opens = gsub(/{/, "{", $0) + closes = gsub(/}/, "}", $0) + depth += (opens - closes) + if (depth <= 0 && $0 ~ /}/) { in_rs=0 } + } + } + ' "/tmp/{{ inventory_hostname }}_config.json" \ + > "/tmp/{{ inventory_hostname }}_config.json.new" + args: + executable: /bin/bash + + - name: Push edited config.json.new to remote temp + when: do_edit + delegate_to: localhost + shell: | + sshpass -p {{ ssh_pass | quote }} scp -q -o StrictHostKeyChecking=no \ + "/tmp/{{ inventory_hostname }}_config.json.new" \ + {{ ssh_user }}@{{ ansible_host }}:/tmp/config.json.new + args: + executable: /bin/bash + + - name: Compute MD5 of local .new + when: do_edit + delegate_to: localhost + command: md5sum "/tmp/{{ inventory_hostname }}_config.json.new" + register: md5_local + changed_when: false + + - name: Compute MD5 of remote .new (no Python) + when: do_edit + raw: "md5sum /tmp/config.json.new || busybox md5sum /tmp/config.json.new" + register: md5_remote + changed_when: false + + - name: Fail if MD5 mismatch + when: + - do_edit + - (md5_local.stdout.split()[0]) != (md5_remote.stdout.split()[0] if (md5_remote.stdout is defined) else 'BAD') + fail: + msg: "MD5 mismatch between controller and remote copy — aborting replace." + + - name: Commit new config.json (overwrite original) + when: do_edit + raw: | + cp -a /tmp/config.json /tmp/config.json.bak.$(date +%Y%m%d%H%M%S) + mv /tmp/config.json.new /tmp/config.json + + - name: Apply config to runtime (sysconf -w) + when: do_edit + raw: | + sysconf -w || /usr/sbin/sysconf -w + register: sysconf_result + changed_when: true + failed_when: sysconf_result.rc not in [0] + + # (d) Pre-restart notice to control queue + - name: Build pre-restart journal (explain why) + when: do_edit + delegate_to: localhost + set_fact: + prereboot_payload: + inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}" + task_name: "journal_add" + task_result: >- + wifidebug: will restart services because remote_syslog settings were updated and applied (sysconf -w). + + - name: Publish pre-restart journal to control queue + when: do_edit + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ prereboot_payload | to_json }}" + payload_encoding: "string" + register: rmq_preboot + changed_when: (rmq_preboot.json is defined) and (rmq_preboot.json.routed | default(false) | bool) + + # ======== SURGICAL CHANGE: restart kick + verification like multissidfix ======== + + - name: Trigger full restart (system-stop; system-start) from controller with 10s cap + when: do_edit + delegate_to: localhost + shell: "timeout 10s sshpass -p {{ ssh_pass | quote }} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=2 -o ServerAliveCountMax=1 {{ ssh_user }}@{{ ansible_host }} '/usr/sbin/system-stop ; sleep 1 ; /usr/sbin/system-start'" + args: + executable: /bin/bash + register: restart_kick + changed_when: true + failed_when: false + + + # probe SSH with nc: up to 24 tries, 5s each (nc timeout -w 3) + - name: Probe SSH with nc (24 tries, 5s each) + when: do_edit + delegate_to: localhost + shell: "nc -z -w 3 {{ ansible_host }} 22" + register: nc_probe + retries: 24 + delay: 5 + until: nc_probe.rc == 0 + changed_when: false + failed_when: false + + - name: Set ssh_up fact (from nc probe) + when: do_edit + delegate_to: localhost + set_fact: + ssh_up: "{{ (nc_probe.rc | default(1)) == 0 }}" + + - name: Fail if SSH did not return after restart + when: + - do_edit + - not ssh_up | bool + fail: + msg: "wifidebug: restart issued; SSH did not return after 24 x 5s checks." + + # ======== END OF SURGICAL CHANGE ======== + + - name: Set result status (success deployed or no change) + set_fact: + result_status: "{{ 'SUCCESS_DEPLOYED' if (upload_wdbg | bool) else 'SUCCESS_NO_CHANGE' }}" + + when: ping_result is succeeded + + - name: Set status fact (no ssh) + when: ping_result is failed + set_fact: + result_status: "NO_SSH" + + rescue: + - name: Mark result as failed + set_fact: + result_status: "FAILED during {{ ansible_failed_task.name }}" + + always: + + - name: Compute inscope device + set_fact: + inscope_device_name: "{{ ansible_hostname | default(inventory_hostname) }}" + + # (a) Set custom field wifidebug -> v15 on control queue + + - name: Build custom-field payload (wifidebug -> version) + delegate_to: localhost + set_fact: + wdbg_cf_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "deploy_wifidebug" + task_result: "{{ wdbg_version }}" + + - name: Publish custom-field update to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wdbg_cf_payload | to_json }}" + payload_encoding: "string" + register: rmq_cf + changed_when: (rmq_cf.json is defined) and (rmq_cf.json.routed | default(false) | bool) + + # (b)(c) Final wrap-up journal "wifidebug: ..." with actions performed + + - name: Build actions list + set_fact: + _actions_list: >- + {{ + [] + + ((upload_wdbg | default(false) | bool) | ternary(['uploaded wifidebug.sh'], [])) + + (((cron_grep is defined) and ((cron_grep.rc | default(0)) != 0)) | ternary(['added crontab entry'], [])) + + ((((sysstart_cron_check.stdout | default('PRESENT')) | trim) != 'PRESENT') | ternary(['updated sysstart.lua (crond launch)'], [])) + + ((do_edit | default(false) | bool) | ternary(['updated remote_syslog + applied config (sysconf -w)'], [])) + + ((do_edit | default(false) | bool) | ternary(['restarted services'], [])) + }} + + - name: Build actions string + set_fact: + _actions_str: "{{ ((_actions_list | default([])) | length > 0) | ternary((_actions_list | join(', ')), 'no changes needed') }}" + + + - name: Build wrap-up journal payload + delegate_to: localhost + set_fact: + wrap_payload: + inscope_device: "{{ inscope_device_name }}" + task_name: "journal_add" + task_result: >- + wifidebug: {{ 'success' if (result_status == 'SUCCESS_DEPLOYED' or result_status == 'SUCCESS_NO_CHANGE') else result_status | lower }} + — actions: {{ _actions_str }} + + - name: Publish wrap-up journal to control queue + delegate_to: localhost + uri: + url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish" + method: POST + user: "{{ rmq_user }}" + password: "{{ rmq_pass }}" + force_basic_auth: true + status_code: 200 + headers: + content-type: "application/json" + body_format: json + body: + properties: + content_type: "application/json" + routing_key: "{{ control_queue }}" + payload: "{{ wrap_payload | to_json }}" + payload_encoding: "string" + register: rmq_wrap + changed_when: (rmq_wrap.json is defined) and (rmq_wrap.json.routed | default(false) | bool) + + # Local summary (kept for operator visibility) + - name: Summary + debug: + msg: + - "result_status: {{ result_status }}" + - "we're good" + diff --git a/files/ansible.cfg b/files/ansible.cfg new file mode 100644 index 0000000..7fccd1a --- /dev/null +++ b/files/ansible.cfg @@ -0,0 +1,14 @@ +[defaults] +inventory = ./inventory/netbox.yml +remote_user = root +ask_pass = false +host_key_checking = False +timeout = 30 +forks = 50 +collections_paths = /opt/containers/ansible-worker/data/collections:/usr/share/ansible/collections + +[inventory] +enable_plugins = auto, yaml, ini, toml, netbox.netbox.nb_inventory + +[ssh_connection] +ssh_args = -o PubkeyAuthentication=no diff --git a/files/entrypoint.sh b/files/entrypoint.sh new file mode 100755 index 0000000..eea2b8f --- /dev/null +++ b/files/entrypoint.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="/opt/containers/ansible-worker" +APP_ROOT="${APP_DIR}/app" +DATA_DIR="${APP_DIR}/data" +SRC_DIR="${APP_DIR}/files" # <-- mounted from host ./files +BIN_DIR="${APP_ROOT}/bin" + +VENV_DIR="${DATA_DIR}/ansible_venv" +COLL_BASE="${DATA_DIR}/collections" +COLL_NETBOX_DIR="${COLL_BASE}/ansible_collections/netbox/netbox" + +mkdir -p "$DATA_DIR" "$BIN_DIR" + +log() { echo "[entrypoint] $*"; } + +copy_nbplay() { + local src="${SRC_DIR}/nbplay" + local dst="${BIN_DIR}/nbplay" + if [[ -f "$src" ]]; then + log "Installing nbplay -> ${dst}" + cp -f "$src" "$dst" + chmod 0755 "$dst" + else + log "WARN: nbplay not found at ${src} (skipping)" + fi +} + +copy_rmq_client() { + local src="${SRC_DIR}/rabbit-client.sh" + local dst="${BIN_DIR}/rabbit-client.sh" + if [[ -f "$src" ]]; then + log "Installing rabbit-client.sh -> ${dst}" + cp -f "$src" "$dst" + chmod 0755 "$dst" + else + log "WARN: rabbit-client.sh not found at ${src} (skipping)" + fi +} + +copy_playbooks() { + local src_dir="${SRC_DIR}/ansible-playbooks" + if [[ -d "$src_dir" ]]; then + log "Syncing playbooks from ${src_dir} -> ${APP_ROOT}" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete "${src_dir}/" "${APP_ROOT}/" + else + # clean only top-level *.yml in /app (playbooks), then copy + find "${APP_ROOT}" -maxdepth 1 -type f -name '*.yml' -exec rm -f {} + + cp -a "${src_dir}/." "${APP_ROOT}/" + fi + else + log "WARN: ${src_dir} does not exist (skipping playbooks copy)" + fi +} + +provision_pwfile() { + local src_yaml="${SRC_DIR}/ssh.yml" + local src_txt="${SRC_DIR}/ssh.txt" + local dst="${DATA_DIR}/ssh.yml" + if [[ -f "$dst" ]]; then + log "Found existing pwfile at ${dst} (leaving as-is)" + return + fi + if [[ -f "$src_yaml" ]]; then + log "Seeding password vars from ${src_yaml} -> ${dst}" + install -m 600 "$src_yaml" "$dst" + return + fi + if [[ -f "$src_txt" ]]; then + log "Wrapping plain password ${src_txt} -> ${dst}" + local pass + pass="$(head -n1 "$src_txt" | tr -d '\r\n')" + umask 177 + { + printf 'ansible_user: %s\n' "${NBPLAY_DEFAULT_USER:-root}" + printf 'ansible_ssh_pass: %s\n' "$pass" + } > "$dst" + return + fi + log "WARN: no files/ssh.yml or files/ssh.txt found; skipping pwfile seed" +} + +# ---- Copy from mounted files/ (if present) ---- +if [[ -d "${SRC_DIR}" ]]; then + log "Found source directory: ${SRC_DIR}" + copy_nbplay + copy_rmq_client + copy_playbooks + provision_pwfile +else + log "WARN: Source directory ${SRC_DIR} not present. Did you mount ./files? (see docker-compose.yml)" +fi + +# ---- Venv bootstrap ---- +if [[ ! -x "${VENV_DIR}/bin/ansible" ]]; then + log "Creating venv in ${VENV_DIR} and installing packages..." + python3 -m venv "${VENV_DIR}" + "${VENV_DIR}/bin/python" -m pip install --upgrade pip setuptools wheel + "${VENV_DIR}/bin/pip" install \ + "ansible-core==2.16.*" \ + "pytz" \ + "pynetbox==7.*" + log "venv bootstrap complete." +fi + +# ---- Collections (persistent path) ---- +if [[ ! -d "${COLL_NETBOX_DIR}" ]]; then + log "Installing netbox.netbox collection into ${COLL_BASE} ..." + mkdir -p "${COLL_BASE}" + "${VENV_DIR}/bin/ansible-galaxy" collection install netbox.netbox -p "${COLL_BASE}" + log "netbox.netbox installed." +fi + +# Put venv first on PATH +export PATH="${VENV_DIR}/bin:${PATH}" + +export TZ=${TZ:-Africa/Johannesburg} + +log "Startup complete. Executing: $*" +exec "$@" diff --git a/files/entrypoint.sh-backup b/files/entrypoint.sh-backup new file mode 100644 index 0000000..9039411 --- /dev/null +++ b/files/entrypoint.sh-backup @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="/opt/containers/ansible-worker" +DATA_DIR="${APP_DIR}/data" +VENV_DIR="${DATA_DIR}/ansible_venv" +COLL_BASE="${DATA_DIR}/collections" +COLL_NETBOX_DIR="${COLL_BASE}/ansible_collections/netbox/netbox" + +mkdir -p "$DATA_DIR" + +# Bootstrap venv on first run +if [ ! -x "${VENV_DIR}/bin/ansible" ]; then + echo "[entrypoint] Creating venv in ${VENV_DIR} and installing packages..." + python3 -m venv "${VENV_DIR}" + "${VENV_DIR}/bin/python" -m pip install --upgrade pip setuptools wheel + "${VENV_DIR}/bin/pip" install \ + "ansible-core==2.16.*" \ + "pytz" \ + "pynetbox==7.*" + echo "[entrypoint] venv bootstrap complete." +fi + +# Ensure NetBox collection exists in the PERSISTENT collections path +if [ ! -d "${COLL_NETBOX_DIR}" ]; then + echo "[entrypoint] Installing netbox.netbox collection into ${COLL_BASE} ..." + mkdir -p "${COLL_BASE}" + "${VENV_DIR}/bin/ansible-galaxy" collection install netbox.netbox -p "${COLL_BASE}" + echo "[entrypoint] netbox.netbox installed." +fi + +# Put venv first on PATH +export PATH="${VENV_DIR}/bin:${PATH}" + +exec "$@" diff --git a/files/files/crond-root b/files/files/crond-root new file mode 100644 index 0000000..6281a2f --- /dev/null +++ b/files/files/crond-root @@ -0,0 +1 @@ +*/1 * * * * /root/wifidebug.sh diff --git a/files/files/crond-root-scroll24 b/files/files/crond-root-scroll24 new file mode 100644 index 0000000..7ee7ca1 --- /dev/null +++ b/files/files/crond-root-scroll24 @@ -0,0 +1 @@ +*/20 * * * * /root/scroll24.sh --allbymyself diff --git a/files/files/filter_1vap_id0.jq b/files/files/filter_1vap_id0.jq new file mode 100644 index 0000000..bbdc7b1 --- /dev/null +++ b/files/files/filter_1vap_id0.jq @@ -0,0 +1,8 @@ +# enable+lbd on index 0 (no touching others) +( . as $root +| ( $root.wireless.radios[$r].vaps // [] ) as $v +| if ($v | length) >= 1 then + .wireless.radios[$r].vaps[0].enabled = true + | .wireless.radios[$r].vaps[0].lbd = true + else . end ) + diff --git a/files/files/filter_1vap_id1.jq b/files/files/filter_1vap_id1.jq new file mode 100644 index 0000000..01bea6d --- /dev/null +++ b/files/files/filter_1vap_id1.jq @@ -0,0 +1,11 @@ +# enable+lbd on index 1 (Policy B) +( . as $root +| ( $root.wireless.radios[$r].vaps // [] ) as $v +| if ($v | length) >= 2 then + .wireless.radios[$r].vaps[1].enabled = true + | .wireless.radios[$r].vaps[1].lbd = true + else + ( .wireless.radios[$r].vaps[1].enabled = true + | .wireless.radios[$r].vaps[1].lbd = true ) + end ) + diff --git a/files/files/filter_2vaps_0_1.jq b/files/files/filter_2vaps_0_1.jq new file mode 100644 index 0000000..50154bc --- /dev/null +++ b/files/files/filter_2vaps_0_1.jq @@ -0,0 +1,9 @@ +# keep 0 (enable+lbd), disable 1 +( . as $root +| ( $root.wireless.radios[$r].vaps // [] ) as $v +| if ($v | length) >= 2 then + .wireless.radios[$r].vaps[0].enabled = true + | .wireless.radios[$r].vaps[0].lbd = true + | .wireless.radios[$r].vaps[1].enabled = false + else . end ) + diff --git a/files/files/filter_2vaps_1_0.jq b/files/files/filter_2vaps_1_0.jq new file mode 100644 index 0000000..3100eb2 --- /dev/null +++ b/files/files/filter_2vaps_1_0.jq @@ -0,0 +1,9 @@ +# layout 1,0 -> same outcome: keep 0 (enable+lbd), disable 1 +( . as $root +| ( $root.wireless.radios[$r].vaps // [] ) as $v +| if ($v | length) >= 2 then + .wireless.radios[$r].vaps[0].enabled = true + | .wireless.radios[$r].vaps[0].lbd = true + | .wireless.radios[$r].vaps[1].enabled = false + else . end ) + diff --git a/files/files/scroll24.sh b/files/files/scroll24.sh new file mode 100644 index 0000000..6db9972 --- /dev/null +++ b/files/files/scroll24.sh @@ -0,0 +1,371 @@ +#!/bin/sh +# wifiscan24.sh — 2.4 GHz channel scout for ath1 (BusyBox-safe) +# Default: probe 1/6/11 (+ current), print recommendation, restore original. +# +# Flags: +# --goforit : scan + apply best, policy prefers 1/6/11, verify +# --force : as above, but pre-reset AP (disable/enable) before applying +# --allbymyself : try like --goforit; if apply fails, auto-bounce AP and retry once +# --keepbadchannels : allow 2–5 or 7–10 if they win (only matters with goforit/force/allbymyself) +# --chan : override — switch directly to channel N (ignores scan & policy) +# works alone (no reset) or with --force (preflight reset) +# --deep : deeper scan (longer dwell); works with apply modes +# +# Score: score = 7*OBSSavg + 3*CUavg (lower is better) +# Ties: lower OBSSmax → lower (more negative) noise → lower CUavg +# +# Apply policy: +# By default apply only to {1,6,11}. To allow middle channels {2..5,7..10} add --keepbadchannels +sleep 20 + +IFACE="${IFACE:-ath1}" +CHANNELS="${CHANNELS:-1 6 11}" +CSA_BEACONS="${CSA_BEACONS:-10}" # CSA duration (~1s at default beacon interval) +WAIT_AFTER_SWITCH="${WAIT_AFTER_SWITCH:-2}" # settle time after CSA during scan/apply +SAMPLES="${SAMPLES:-5}" # samples per channel +SLEEP_BETWEEN="${SLEEP_BETWEEN:-1}" # seconds between samples + +# Deep-scan presets (used only with --deep) +DEEP_WAIT="2" +DEEP_SAMPLES="10" +DEEP_SLEEP="1" + +GOOD_CH_SET="1 6 11" # policy-preferred set + +GOFORIT=0 +FORCE=0 +ALLBY=0 +DEEP=0 +KEEPBAD=0 +OVERRIDE_CH="" + +# ---- arg parse ---- +while [ $# -gt 0 ]; do + case "$1" in + --goforit|--go|--apply) GOFORIT=1 ;; + --force) FORCE=1 ;; + --allbymyself) ALLBY=1 ;; + --deep) DEEP=1 ;; + --keepbadchannels) KEEPBAD=1 ;; + --chan) + shift + [ -n "$1" ] || { echo "Error: --chan requires a channel number." >&2; exit 1; } + OVERRIDE_CH="$1" + ;; + -h|--help) + cat <&2 + exit 1 + ;; + esac + shift +done + +# If multiple apply flags passed, pick strongest precedence: FORCE > ALLBY > GOFORIT +if [ "$FORCE" -eq 1 ]; then + GOFORIT=0 + ALLBY=0 +elif [ "$ALLBY" -eq 1 ]; then + GOFORIT=0 +fi + +# Deep dwell +if [ "$DEEP" -eq 1 ]; then + WAIT_AFTER_SWITCH="$DEEP_WAIT" + SAMPLES="$DEEP_SAMPLES" + SLEEP_BETWEEN="$DEEP_SLEEP" +fi + +# ---- helpers ---- +freq_from_ch() { ch="$1"; [ "$ch" -eq 14 ] && { echo 2484; return; }; echo $((2412 + 5*(ch-1))); } +ch_from_freq() { f="$1"; [ "$f" -eq 2484 ] && { echo 14; return; }; echo $(( (f-2412)/5 + 1 )); } + +valid_ch() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + *) [ "$1" -ge 1 ] && [ "$1" -le 14 ] ;; + esac +} + +in_good_set() { + case " $GOOD_CH_SET " in + *" $1 "*) return 0 ;; + *) return 1 ;; + esac +} + +get_current_freq() { + f="$(hostapd_cli -i "$IFACE" status 2>/dev/null | awk -F= '/^freq=/{print $2; exit}')" + [ -n "$f" ] || f="$(wifitool "$IFACE" get_rrmutil 2>/dev/null | awk -F': *' '/^channel/{print $2; exit}')" + echo "$f" +} + +measure_once() { + # prints: "chan_util obss_util noise" + wifitool "$IFACE" get_rrmutil 2>/dev/null | awk -F': *' ' + /^chan util/ {cu=$2+0} + /^obss util/ {ob=$2+0} + /^noise floor/ {nf=$2+0} + END {printf "%d %d %d\n", cu, ob, nf} + ' +} + +# CSA wrapper (quiet) +do_csa() { + hostapd_cli -i "$IFACE" chan_switch "$CSA_BEACONS" "$1" ht bandwidth=20 blocktx >/dev/null 2>&1 +} + +# Verify helper +verify_freq() { + target="$1" + sleep "$WAIT_AFTER_SWITCH" + cur="$(get_current_freq)" + [ "$cur" = "$target" ] +} + +# Self-bounce helper +ap_bounce() { + hostapd_cli -i "$IFACE" disable >/dev/null 2>&1 + sleep 2 + hostapd_cli -i "$IFACE" enable >/dev/null 2>&1 + sleep 2 +} + +# ---- prereqs ---- +command -v hostapd_cli >/dev/null 2>&1 || { echo "hostapd_cli not found"; exit 1; } +command -v wifitool >/dev/null 2>&1 || { echo "wifitool not found"; exit 1; } + +orig_freq="$(get_current_freq)"; orig_ch="$(ch_from_freq "$orig_freq")" + +# ---------- OVERRIDE PATH: --chan N ---------- +if [ -n "$OVERRIDE_CH" ]; then + valid_ch "$OVERRIDE_CH" || { echo "Invalid channel: $OVERRIDE_CH (expected 1..14)"; exit 1; } + target_freq="$(freq_from_ch "$OVERRIDE_CH")" + if [ "$target_freq" = "$orig_freq" ]; then + echo "Already on channel $OVERRIDE_CH ($target_freq MHz). No switch needed." + exit 0 + fi + + if [ "$FORCE" -eq 1 ]; then + echo "Pre-resetting AP before override…" + ap_bounce + fi + + echo "Applying override: hostapd_cli -i $IFACE chan_switch $CSA_BEACONS $target_freq ht bandwidth=20 blocktx" + do_csa "$target_freq" + if verify_freq "$target_freq"; then + echo "Switched to ch $OVERRIDE_CH ($target_freq MHz)." + exit 0 + else + echo "Override failed to take effect. Try: --chan $OVERRIDE_CH --force" + exit 1 + fi +fi + +# ---------- SCAN PATH ---------- +# unique set incl. current +unique=""; add(){ case " $unique " in *" $1 "*) : ;; *) unique="$unique $1";; esac; } +for c in $CHANNELS; do add "$c"; done; add "$orig_ch" + +echo "Interface: $IFACE" +echo "Original: ch $orig_ch ($orig_freq MHz)" +[ "$DEEP" -eq 1 ] && echo "Mode: deep scan (WAIT=$WAIT_AFTER_SWITCH s, SAMPLES=$SAMPLES, SLEEP=$SLEEP_BETWEEN s)" +echo "Testing: $unique" +echo + +tmp="$(mktemp)" +RESTORE=1 +cleanup() { + if [ "$RESTORE" = 1 ]; then + do_csa "$orig_freq" + fi + rm -f "$tmp" +} +trap 'cleanup' EXIT INT TERM + +measure_channel() { + ch="$1"; freq="$(freq_from_ch "$ch")" + curf="$(get_current_freq)" + if [ "$curf" != "$freq" ]; then + do_csa "$freq" + sleep "$WAIT_AFTER_SWITCH" + fi + + cu_min=9999; cu_max=0; cu_sum=0 + ob_min=9999; ob_max=0; ob_sum=0 + nf_min=9999; nf_max=-9999; nf_sum=0 + + i=0 + while [ "$i" -lt "$SAMPLES" ]; do + set -- $(measure_once) # $1=cu $2=ob $3=nf + cu="$1"; ob="$2"; nf="$3" + + [ "$cu" -lt "$cu_min" ] && cu_min="$cu"; [ "$cu" -gt "$cu_max" ] && cu_max="$cu"; cu_sum=$((cu_sum+cu)) + [ "$ob" -lt "$ob_min" ] && ob_min="$ob"; [ "$ob" -gt "$ob_max" ] && ob_max="$ob"; ob_sum=$((ob_sum+ob)) + [ "$nf" -lt "$nf_min" ] && nf_min="$nf"; [ "$nf" -gt "$nf_max" ] && nf_max="$nf"; nf_sum=$((nf_sum+nf)) + + i=$((i+1)) + [ "$i" -lt "$SAMPLES" ] && sleep "$SLEEP_BETWEEN" + done + + cu_avg=$((cu_sum / SAMPLES)); ob_avg=$((ob_sum / SAMPLES)); nf_avg=$((nf_sum / SAMPLES)) + + printf "ch %2d (%4d MHz): OBSS min/avg/max=%3d/%3d/%3d CHutil min/avg/max=%3d/%3d/%3d Noise avg=%4d dBm\n" \ + "$ch" "$freq" "$ob_min" "$ob_avg" "$ob_max" "$cu_min" "$cu_avg" "$cu_max" "$nf_avg" + + # machine line: ch ob_avg cu_avg ob_max nf_avg + echo "__RESULT__ $ch $ob_avg $cu_avg $ob_max $nf_avg" +} + +# Measure all +for ch in $unique; do + out="$(measure_channel "$ch")" + printf "%s\n" "$out" | head -n 1 + printf "%s\n" "$out" | awk '/^__RESULT__/ {print}' >>"$tmp" +done + +# Pick best overall and best among 1/6/11 +best_any_ch=""; best_any_score=999999; best_any_obmax=9999; best_any_nf=0; best_any_cu=9999 +best_good_ch=""; best_good_score=999999; best_good_obmax=9999; best_good_nf=0; best_good_cu=9999 + +while read -r _ ch ob cu obmax nf; do + score=$(( 7*ob + 3*cu )) + + # overall best + if [ "$score" -lt "$best_any_score" ]; then + best_any_score="$score"; best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu" + elif [ "$score" -eq "$best_any_score" ]; then + # Tie-breaker without command groups — ash-safe + if [ "$obmax" -lt "$best_any_obmax" ]; then + best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu" + elif [ "$obmax" -eq "$best_any_obmax" ]; then + if [ "$nf" -lt "$best_any_nf" ]; then + best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu" + elif [ "$nf" -eq "$best_any_nf" ] && [ "$cu" -lt "$best_any_cu" ]; then + best_any_ch="$ch"; best_any_obmax="$obmax"; best_any_nf="$nf"; best_any_cu="$cu" + fi + fi + fi + + # best among 1/6/11 + if in_good_set "$ch"; then + if [ "$score" -lt "$best_good_score" ]; then + best_good_score="$score"; best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu" + elif [ "$score" -eq "$best_good_score" ]; then + # Tie-breaker without command groups — ash-safe + if [ "$obmax" -lt "$best_good_obmax" ]; then + best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu" + elif [ "$obmax" -eq "$best_good_obmax" ]; then + if [ "$nf" -lt "$best_good_nf" ]; then + best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu" + elif [ "$nf" -eq "$best_good_nf" ] && [ "$cu" -lt "$best_good_cu" ]; then + best_good_ch="$ch"; best_good_obmax="$obmax"; best_good_nf="$nf"; best_good_cu="$cu" + fi + fi + fi + fi +done <"$tmp" + +best_any_freq="$(freq_from_ch "$best_any_ch")" +best_good_freq=""; [ -n "$best_good_ch" ] && best_good_freq="$(freq_from_ch "$best_good_ch")" + +echo +echo ">>> Best by score: ch $best_any_ch ($best_any_freq MHz) [score=${best_any_score} = 7*OBSSavg + 3*CUavg]" + +# Apply target by policy +apply_ch=""; apply_freq=""; note="policy prefers 1/6/11" +if [ "$KEEPBAD" -eq 1 ]; then + apply_ch="$best_any_ch"; apply_freq="$best_any_freq"; note="--keepbadchannels set (allow 2–5/7–10)" +else + if [ -n "$best_good_ch" ]; then + apply_ch="$best_good_ch"; apply_freq="$best_good_freq" + else + apply_ch="$best_any_ch"; apply_freq="$best_any_freq"; note="no scorable 1/6/11; falling back to overall best" + fi +fi + +echo ">>> Policy target (apply): ch $apply_ch ($apply_freq MHz) [$note]" +[ "$apply_freq" = "$orig_freq" ] && APPLY_SAME=1 || APPLY_SAME=0 + +# ---------- APPLY SECTION ---------- +if [ "$GOFORIT" -eq 1 ] || [ "$FORCE" -eq 1 ] || [ "$ALLBY" -eq 1 ]; then + if [ "$APPLY_SAME" -eq 1 ]; then + echo "Already on the policy target channel. No switch needed." + RESTORE=0 + exit 0 + fi +fi + +# Common apply attempt +try_apply_once() { + tgt="$1" + echo "Applying (no reset): hostapd_cli -i $IFACE chan_switch $CSA_BEACONS $tgt ht bandwidth=20 blocktx" + do_csa "$tgt" + if verify_freq "$tgt"; then + return 0 + fi + return 1 +} + +if [ "$FORCE" -eq 1 ]; then + echo "Pre-resetting AP (force)…" + ap_bounce + echo "Applying after reset: hostapd_cli -i $IFACE chan_switch $CSA_BEACONS $apply_freq ht bandwidth=20 blocktx" + do_csa "$apply_freq" + if verify_freq "$apply_freq"; then + echo "Switched to ch $apply_ch ($apply_freq MHz) after AP reset." + RESTORE=0 + exit 0 + else + echo "Unable to switch automatically at this time. Keeping the original channel." + exit 1 + fi +elif [ "$ALLBY" -eq 1 ]; then + # First: normal CSA + if try_apply_once "$apply_freq"; then + echo "Switched to ch $apply_ch ($apply_freq MHz)." + RESTORE=0 + exit 0 + fi + # Second: bounce then retry once + echo "Apply failed; self-bouncing AP and retrying once…" + ap_bounce + if try_apply_once "$apply_freq"; then + echo "Switched to ch $apply_ch ($apply_freq MHz) after self-bounce." + RESTORE=0 + exit 0 + fi + echo "Retry after self-bounce also failed. Keeping the original channel." + exit 1 +elif [ "$GOFORIT" -eq 1 ]; then + if try_apply_once "$apply_freq"; then + echo "Switched to ch $apply_ch ($apply_freq MHz)." + RESTORE=0 + exit 0 + fi + echo "Cannot switch now; the radio needs a brief reset. Re-run with --allbymyself or --force${KEEPBAD:+ --keepbadchannels}." + exit 1 +fi + +# ---------- SCAN-ONLY ---------- +echo "Scan-only mode." +echo "To apply (prefer 1/6/11): $0 --goforit" +echo "To self-heal on failure: $0 --allbymyself" +echo "To apply with reset: $0 --force" +echo "To allow middle chans: add --keepbadchannels with one of the above" +echo "To override to a chan: $0 --chan N (optionally add --force)" diff --git a/files/files/wifidebug.sh b/files/files/wifidebug.sh new file mode 100644 index 0000000..6c69dc7 --- /dev/null +++ b/files/files/wifidebug.sh @@ -0,0 +1,421 @@ +#!/bin/sh + +#sleep $((RANDOM % 18 + 3)) + +SCRIPTVERSION=17 + +delay=$(( ( $(hexdump -n2 -e '/2 "%u"' /dev/urandom) % 18 ) + 3 )) +#echo "Sleeping for $delay seconds..." +sleep $delay + +# Get hostname +HOSTNAME=$(cat /proc/sys/kernel/hostname) + +FWVER=$(cat /etc/banner | grep rev | awk '{ print $1 }') + +# =========================================================== +# GPS coordinates management (locks, timeout, 0/0 fallback) +# - Writes /tmp/gps_coordinates with: +# longitude: +# latitude: +# - Uses /tmp/gps_coordinates..lock (expires 10m) +# - Tracks retries in /tmp/gps_timeout (stop after 3) +# - On any retrieval attempt (success/fail) -> exit cycle +# =========================================================== + +COORD_FILE="/tmp/gps_coordinates" +TIMEOUT_FILE="/tmp/gps_timeout" +LOCK_MAX_AGE_MIN=10 # minutes +RETRIEVE_MAX_WAIT=480 # seconds (8 minutes) +LOCATION_LON="0.000000" +LOCATION_LAT="0.000000" + +# Helper: parse existing coords file (if any) +read_coords_file() { + if [ -f "$COORD_FILE" ]; then + # Expect two lines: "longitude: X" and "latitude: Y" + local lon lat + lon=$(awk -F': *' '/^longitude:/ {print $2; exit}' "$COORD_FILE") + lat=$(awk -F': *' '/^latitude:/ {print $2; exit}' "$COORD_FILE") + # Validate numeric and ranges; format to 6 decimals + if echo "$lon" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then + if echo "$lat" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then + if awk -v a="$lat" -v b="$lon" 'BEGIN{exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}'; then + LOCATION_LON=$(awk -v x="$lon" 'BEGIN{printf "%.6f", x+0}') + LOCATION_LAT=$(awk -v x="$lat" 'BEGIN{printf "%.6f", x+0}') + return 0 + fi + fi + fi + fi + return 1 +} + +# If coords exist and valid -> proceed; else attempt retrieval logic +if read_coords_file; then + : +else + # Check retry threshold + GTO=0 + if [ -f "$TIMEOUT_FILE" ]; then + GTO=$(cat "$TIMEOUT_FILE" 2>/dev/null | awk '/^[0-9]+$/ {print $0; exit}') + [ -z "$GTO" ] && GTO=0 + fi + + if [ "$GTO" -ge 3 ]; then + # Give up trying; ensure coords file exists with 0/0 and exit this cycle + printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp" + mv "${COORD_FILE}.tmp" "$COORD_FILE" + exit 0 + fi + + # Clean up stale locks (>10 minutes) + find /tmp -maxdepth 1 -type f -name 'gps_coordinates.*.lock' -mmin +"$LOCK_MAX_AGE_MIN" -exec rm -f {} \; 2>/dev/null + + # If any non-stale lock remains, skip this run + for f in /tmp/gps_coordinates.*.lock; do + [ -e "$f" ] || continue + # Found an active (<=10m) lock; skip this cycle + exit 0 + done + + # Create fresh lock and start retrieval + NOWTS=$(date +%s) + LOCK_FILE="/tmp/gps_coordinates.${NOWTS}.lock" + : > "$LOCK_FILE" 2>/dev/null + + CAND="/tmp/gps_candidate.$$" + ( + gpspipe -r | awk -F, ' + # -------- THE ONLY CHANGE IS HERE: use deglen+1 for minutes -------- + function dm2dd(dm, hemi, deglen, deg, min, dd) { + if (dm=="" || hemi=="") return "" + deglen = (hemi=="N" || hemi=="S") ? 2 : 3 + deg = substr(dm, 1, deglen) + 0 + min = substr(dm, deglen+1) + 0 + dd = deg + (min / 60.0) + return (hemi=="S" || hemi=="W") ? -dd : dd + } + # Wait for first valid RMC (A) + /^\$..RMC/ && $3=="A" { + lat = dm2dd($4, $5) + lon = dm2dd($6, $7) + if (lat != "" && lon != "") { + printf("%.6f %.6f\n", lon, lat) + exit + } + } + ' > "$CAND" + ) & + GPID=$! + + # Wait up to RETRIEVE_MAX_WAIT seconds for candidate to appear + waited=0 + while [ "$waited" -lt "$RETRIEVE_MAX_WAIT" ]; do + if [ -s "$CAND" ]; then + break + fi + sleep 1 + waited=$((waited+1)) + done + + success=0 + if [ -s "$CAND" ]; then + set -- $(head -n1 "$CAND" 2>/dev/null) + cand_lon="$1" + cand_lat="$2" + if awk -v a="$cand_lat" -v b="$cand_lon" 'BEGIN{ok=(a!="" && b!=""); if (!ok) exit 1; exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}' + then + printf "longitude: %.6f\nlatitude: %.6f\n" "$cand_lon" "$cand_lat" > "${COORD_FILE}.tmp" + mv "${COORD_FILE}.tmp" "$COORD_FILE" + echo 0 > "$TIMEOUT_FILE" + success=1 + fi + fi + + # Cleanup background gpspipe if still running + kill "$GPID" 2>/dev/null + sleep 1 + kill -9 "$GPID" 2>/dev/null + + # Remove lock and temp + rm -f "$LOCK_FILE" "$CAND" 2>/dev/null + + # Exit this cycle after any retrieval attempt (success or fail) + if [ "$success" -eq 1 ]; then + exit 0 + else + printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp" + mv "${COORD_FILE}.tmp" "$COORD_FILE" + GTO=$((GTO+1)) + echo "$GTO" > "$TIMEOUT_FILE" + exit 0 + fi +fi + +# At this point, LOCATION_LON/LOCATION_LAT are set (from file or default 0/0) + +# Get OBSS utilization value from wifitool +ATH0_OBSS_UTIL=$(wifitool ath0 get_rrmutil | awk '/obss util/ { print $4 }') +ATH1_OBSS_UTIL=$(wifitool ath1 get_rrmutil | awk '/obss util/ { print $4 }') +ATH0_CH_UTIL=$(cfg80211tool ath0 get_chutil | awk -F ':' '{ print $2 }') +ATH1_CH_UTIL=$(cfg80211tool ath1 get_chutil | awk -F ':' '{ print $2 }') + +# ----------------------------------------------------------- +# Count connected (real) clients on ath0 and ath1 +# ----------------------------------------------------------- + +get_usercount() { + IFACE=$1 + hostapd_cli -i "$IFACE" all_sta 2>/dev/null | awk ' + /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/ {mac=$0; next} + /flags=/ { + auth = (index($0, "AUTH") > 0) + assoc = (index($0, "ASSOC") > 0) + authorized = (index($0, "AUTHORIZED") > 0) + } + /rx_packets=/ {rx_packets=$1; sub("rx_packets=", "", rx_packets)} + /tx_packets=/ {tx_packets=$1; sub("tx_packets=", "", tx_packets)} + /inactive_msec=/ {inactive=$1; sub("inactive_msec=", "", inactive)} + /connected_time=/ { + connected=$1; sub("connected_time=", "", connected) + if (auth && assoc && authorized && + rx_packets+0 > 0 && tx_packets+0 > 0 && + inactive+0 < 5000 && connected+0 > 60) { + count++ + } + } + END { print count+0 } + ' +} + +ATH0_USERCOUNT=$(get_usercount ath0) +ATH1_USERCOUNT=$(get_usercount ath1) + +# ----------------------------------------------------------- +# Get IP/MAC for br-wan +# ----------------------------------------------------------- + +MACADDRESS=$(ip link show br-wan | awk '/link\/ether/ {print $2}') +IPADDR_MASK=$(ip -4 addr show br-wan | awk '/inet / {print $2}' | head -n 1) +IPADDRESS=${IPADDR_MASK%/*} +IPMASK=${IPADDR_MASK#*/} + +# ----------------------------------------------------------- +# Get LLDP neighbor switch +NEIGH_SW=$(grep ikeja /tmp/run/lldp_server.json | grep -E "\-sc|\-as" -m1 | sed -E 's/.*"system_name": "([^"]+)".*/\1/') + +# ----------------------------------------------------------- +# Get default GW +DEFGW=$(ip route show | grep default | awk '{ print $3 }') + +# ----------------------------------------------------------- + +# Run once per iface and reuse the buffer +ATH0_RAW="$(apstats -v -i ath0 2>/dev/null)" +ATH1_RAW="$(apstats -v -i ath1 2>/dev/null)" + +# ---- ath0 (read from ATH0_RAW) ---- +ATH0_TXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }') +ATH0_RXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }') +ATH0_TXUDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }') +ATH0_TXMBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }') +ATH0_TXMDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }') +ATH0_TXBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }') +ATH0_TXFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Tx failures" | awk '{ print $NF }') +ATH0_PCKERR=$(printf '%s\n' "$ATH0_RAW" | grep "Packets Errored" | awk '{ print $NF }') +ATH0_RETR=$(printf '%s\n' "$ATH0_RAW" | grep "Retries" | awk '{ print $NF }') +ATH0_BCNSUC=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon success" | awk '{ print $NF }') +ATH0_BCNFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon failed" | awk '{ print $NF }') + +ATH0_TXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') +ATH0_RXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') + +# ---- ath1 (read from ATH1_RAW) ---- +ATH1_TXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }') +ATH1_RXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }') +ATH1_TXUDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }') +ATH1_TXMBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }') +ATH1_TXMDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }') +ATH1_TXBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }') +ATH1_TXFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Tx failures" | awk '{ print $NF }') +ATH1_PCKERR=$(printf '%s\n' "$ATH1_RAW" | grep "Packets Errored" | awk '{ print $NF }') +ATH1_RETR=$(printf '%s\n' "$ATH1_RAW" | grep "Retries" | awk '{ print $NF }') +ATH1_BCNSUC=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon success" | awk '{ print $NF }') +ATH1_BCNFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon failed" | awk '{ print $NF }') + +ATH1_TXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') +ATH1_RXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') + +# ----------------------------------------------------------- +# channels +# ----------------------------------------------------------- + +ATH0_CH_NMBR=$(iw dev ath0 info | grep MHz | awk '{ print $2 }') +ATH0_CH_WIDTH=$(iw dev ath0 info | grep MHz | awk '{ print $6 }') +ATH0_CH_CNTR=$(iw dev ath0 info | grep MHz | awk '{ print $(NF-1) }') + +ATH1_CH_NMBR=$(iw dev ath1 info | grep MHz | awk '{ print $2 }') +ATH1_CH_WIDTH=$(iw dev ath1 info | grep MHz | awk '{ print $6 }') +ATH1_CH_CNTR=$(iw dev ath1 info | grep MHz | awk '{ print $(NF-1) }') + +# --- ath1 RSSI buckets (gold/silver/bronze/trash) --- +ATH1_USGLD=0; ATH1_USSLV=0; ATH1_USBRN=0; ATH1_USTRS=0 +_assigns="$( + ( + exec 2>/dev/null + hostapd_cli -i ath1 all_sta \ + | grep 'AUTHORIZED' -B1 -A12 \ + | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \ + | awk ' + BEGIN { g=0; s=0; b=0; t=0 } + /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next } + /^signal=/ { + sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0 + if (sig>=-65 && sig<=-20) g++ + else if (sig>=-75 && sig<=-66) s++ + else if (sig>=-85 && sig<=-76) b++ + else if (sig>=-200 && sig<=-86) t++ + } + END { + printf "ATH1_USGLD=%d\nATH1_USSLV=%d\nATH1_USBRN=%d\nATH1_USTRS=%d\n", g+0, s+0, b+0, t+0 + } + ' + ) || true +)" +if printf '%s\n' "$_assigns" | awk -F= ' + NR==1 && $1=="ATH1_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1} + NR==2 && $1=="ATH1_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1} + NR==3 && $1=="ATH1_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1} + NR==4 && $1=="ATH1_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1} + END{ exit !(ok1&&ok2&&ok3&&ok4) } +'; then + eval "$_assigns" +fi +case "$ATH1_USGLD" in (''|*[!0-9]*) ATH1_USGLD=0;; esac +case "$ATH1_USSLV" in (''|*[!0-9]*) ATH1_USSLV=0;; esac +case "$ATH1_USBRN" in (''|*[!0-9]*) ATH1_USBRN=0;; esac +case "$ATH1_USTRS" in (''|*[!0-9]*) ATH1_USTRS=0;; esac + +# --- ath0 RSSI buckets (gold/silver/bronze/trash) --- +ATH0_USGLD=0; ATH0_USSLV=0; ATH0_USBRN=0; ATH0_USTRS=0 +_assigns="$( + ( + exec 2>/dev/null + hostapd_cli -i ath0 all_sta \ + | grep 'AUTHORIZED' -B1 -A12 \ + | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \ + | awk ' + BEGIN { g=0; s=0; b=0; t=0 } + /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next } + /^signal=/ { + sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0 + if (sig>=-65 && sig<=-20) g++ + else if (sig>=-75 && sig<=-66) s++ + else if (sig>=-85 && sig<=-76) b++ + else if (sig>=-200 && sig<=-86) t++ + } + END { + printf "ATH0_USGLD=%d\nATH0_USSLV=%d\nATH0_USBRN=%d\nATH0_USTRS=%d\n", g+0, s+0, b+0, t+0 + } + ' + ) || true +)" +if printf '%s\n' "$_assigns" | awk -F= ' + NR==1 && $1=="ATH0_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1} + NR==2 && $1=="ATH0_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1} + NR==3 && $1=="ATH0_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1} + NR==4 && $1=="ATH0_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1} + END{ exit !(ok1&&ok2&&ok3&&ok4) } +'; then + eval "$_assigns" +fi +case "$ATH0_USGLD" in (''|*[!0-9]*) ATH0_USGLD=0;; esac +case "$ATH0_USSLV" in (''|*[!0-9]*) ATH0_USSLV=0;; esac +case "$ATH0_USBRN" in (''|*[!0-9]*) ATH0_USBRN=0;; esac +case "$ATH0_USTRS" in (''|*[!0-9]*) ATH0_USTRS=0;; esac + +# ----------------------------------------------------------- +# FCS totals/errored +# ----------------------------------------------------------- + +WIFISTATS_ATH0_FCSOK=$(wifistats wifi0 2 | grep fcs_ok | awk '{ print $3 }') +WIFISTATS_ATH0_FCSERR=$(wifistats wifi0 2 | grep fcs_err | awk '{ print $3 }') + +WIFISTATS_ATH1_FCSOK=$(wifistats wifi1 2 | grep fcs_ok | awk '{ print $3 }') +WIFISTATS_ATH1_FCSERR=$(wifistats wifi1 2 | grep fcs_err | awk '{ print $3 }') + +# 10-hex-char uuid for correlation (shared between the two parts) +UUID=$(hexdump -n5 -e '5/1 "%02x"' /dev/urandom) + +# ------------------------ +# Part 1: ath0 + shared (+ GPS) +# ------------------------ +MSG1="hostname=${HOSTNAME} \ +uuid=${UUID} part=1 \ +scrver=${SCRIPTVERSION} \ +ath0_ch_nmbr=${ATH0_CH_NMBR} \ +ath0_ch_width=${ATH0_CH_WIDTH} \ +ath0_ch_cntr=${ATH0_CH_CNTR} \ +ath0_obss_util=${ATH0_OBSS_UTIL} \ +ath0_chutil=${ATH0_CH_UTIL} \ +ath0_txdp=${ATH0_TXDP} \ +ath0_rxdp=${ATH0_RXDP} \ +ath0_txudp=${ATH0_TXUDP} \ +ath0_txmbdp=${ATH0_TXMBDP} \ +ath0_txmdp=${ATH0_TXMDP} \ +ath0_txbdp=${ATH0_TXBDP} \ +ath0_txflr=${ATH0_TXFLR} \ +ath0_txrate=${ATH0_TXRATE} \ +ath0_rxrate=${ATH0_RXRATE} \ +ath0_pckerr=${ATH0_PCKERR} \ +ath0_retr=${ATH0_RETR} \ +ath0_bcnsuc=${ATH0_BCNSUC} \ +ath0_bcnflr=${ATH0_BCNFLR} \ +ath0_usercount=${ATH0_USERCOUNT} \ +ath0_usgld=${ATH0_USGLD} \ +ath0_usslv=${ATH0_USSLV} \ +ath0_usbrn=${ATH0_USBRN} \ +ath0_ustrs=${ATH0_USTRS} \ +ath0_fcsok=${WIFISTATS_ATH0_FCSOK} \ +ath0_fcserr=${WIFISTATS_ATH0_FCSERR} \ +fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW} \ +location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}" + +# ------------------------ +# Part 2: ath1 only +# ------------------------ +MSG2="hostname=${HOSTNAME} \ +uuid=${UUID} part=2 \ +scrver=${SCRIPTVERSION} \ +ath1_ch_nmbr=${ATH1_CH_NMBR} \ +ath1_ch_width=${ATH1_CH_WIDTH} \ +ath1_ch_cntr=${ATH1_CH_CNTR} \ +ath1_obss_util=${ATH1_OBSS_UTIL} \ +ath1_chutil=${ATH1_CH_UTIL} \ +ath1_txdp=${ATH1_TXDP} \ +ath1_rxdp=${ATH1_RXDP} \ +ath1_txudp=${ATH1_TXUDP} \ +ath1_txmbdp=${ATH1_TXMBDP} \ +ath1_txmdp=${ATH1_TXMDP} \ +ath1_txbdp=${ATH1_TXBDP} \ +ath1_txflr=${ATH1_TXFLR} \ +ath1_txrate=${ATH1_TXRATE} \ +ath1_rxrate=${ATH1_RXRATE} \ +ath1_pckerr=${ATH1_PCKERR} \ +ath1_retr=${ATH1_RETR} \ +ath1_bcnsuc=${ATH1_BCNSUC} \ +ath1_bcnflr=${ATH1_BCNFLR} \ +ath1_usercount=${ATH1_USERCOUNT} \ +ath1_usgld=${ATH1_USGLD} \ +ath1_usslv=${ATH1_USSLV} \ +ath1_usbrn=${ATH1_USBRN} \ +ath1_ustrs=${ATH1_USTRS} \ +ath1_fcsok=${WIFISTATS_ATH1_FCSOK} \ +ath1_fcserr=${WIFISTATS_ATH1_FCSERR} \ +fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW} \ +location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}" + +# Send messages via syslog (two separate lines, same tag) +logger -t "debug|${HOSTNAME}" "$MSG1" +logger -t "debug|${HOSTNAME}" "$MSG2" diff --git a/files/files/wifidebug12.sh b/files/files/wifidebug12.sh new file mode 100644 index 0000000..f2e6c37 --- /dev/null +++ b/files/files/wifidebug12.sh @@ -0,0 +1,294 @@ +#!/bin/sh + +#sleep $((RANDOM % 18 + 3)) + +SCRIPTVERSION=12 + +delay=$(( ( $(hexdump -n2 -e '/2 "%u"' /dev/urandom) % 18 ) + 3 )) +#echo "Sleeping for $delay seconds..." +sleep $delay + +# Get hostname +HOSTNAME=$(cat /proc/sys/kernel/hostname) + +FWVER=$(cat /etc/banner | grep rev | awk '{ print $1 }') + +# Get OBSS utilization value from wifitool +ATH0_OBSS_UTIL=$(wifitool ath0 get_rrmutil | awk '/obss util/ { print $4 }') +ATH1_OBSS_UTIL=$(wifitool ath1 get_rrmutil | awk '/obss util/ { print $4 }') +ATH0_CH_UTIL=$(cfg80211tool ath0 get_chutil | awk -F ':' '{ print $2 }') +ATH1_CH_UTIL=$(cfg80211tool ath1 get_chutil | awk -F ':' '{ print $2 }') + +# ----------------------------------------------------------- +# Count connected (real) clients on ath0 and ath1 +# ----------------------------------------------------------- + +get_usercount() { + IFACE=$1 + hostapd_cli -i "$IFACE" all_sta 2>/dev/null | awk ' + /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/ {mac=$0; next} + /flags=/ { + auth = (index($0, "AUTH") > 0) + assoc = (index($0, "ASSOC") > 0) + authorized = (index($0, "AUTHORIZED") > 0) + } + /rx_packets=/ {rx_packets=$1; sub("rx_packets=", "", rx_packets)} + /tx_packets=/ {tx_packets=$1; sub("tx_packets=", "", tx_packets)} + /inactive_msec=/ {inactive=$1; sub("inactive_msec=", "", inactive)} + /connected_time=/ { + connected=$1; sub("connected_time=", "", connected) + if (auth && assoc && authorized && + rx_packets+0 > 0 && tx_packets+0 > 0 && + inactive+0 < 5000 && connected+0 > 60) { + count++ + } + } + END { print count+0 } + ' +} + +ATH0_USERCOUNT=$(get_usercount ath0) +ATH1_USERCOUNT=$(get_usercount ath1) + +# ----------------------------------------------------------- +# Get IP/MAC for br-wan +# ----------------------------------------------------------- + +# MAC address +MACADDRESS=$(ip link show br-wan | awk '/link\/ether/ {print $2}') + +# IP address and mask (CIDR) +IPADDR_MASK=$(ip -4 addr show br-wan | awk '/inet / {print $2}' | head -n 1) + +# Split into IP and mask separately (optional) +IPADDRESS=${IPADDR_MASK%/*} +IPMASK=${IPADDR_MASK#*/} + +# ----------------------------------------------------------- +# Get LLDP neighbor switch +# + +NEIGH_SW=$(grep ikeja /tmp/run/lldp_server.json | grep -E "\-sc|\-as" -m1 | sed -E 's/.*"system_name": "([^"]+)".*/\1/') + +# ----------------------------------------------------------- +# Get default GW +# ----------------------------------------------------------- + +DEFGW=$(ip route show | grep default | awk '{ print $3 }') + +# ----------------------------------------------------------- + +# Run once per iface and reuse the buffer +ATH0_RAW="$(apstats -v -i ath0 2>/dev/null)" +ATH1_RAW="$(apstats -v -i ath1 2>/dev/null)" + +# ---- ath0 (read from ATH0_RAW) ---- +ATH0_TXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }') +ATH0_RXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }') +ATH0_TXUDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }') +ATH0_TXMBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }') +ATH0_TXMDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }') +ATH0_TXBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }') +ATH0_TXFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Tx failures" | awk '{ print $NF }') +ATH0_PCKERR=$(printf '%s\n' "$ATH0_RAW" | grep "Packets Errored" | awk '{ print $NF }') +ATH0_RETR=$(printf '%s\n' "$ATH0_RAW" | grep "Retries" | awk '{ print $NF }') +ATH0_BCNSUC=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon success" | awk '{ print $NF }') +ATH0_BCNFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon failed" | awk '{ print $NF }') + +ATH0_TXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') +ATH0_RXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') + + +# ---- ath1 (read from ATH1_RAW) ---- +ATH1_TXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }') +ATH1_RXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }') +ATH1_TXUDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }') +ATH1_TXMBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }') +ATH1_TXMDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }') +ATH1_TXBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }') +ATH1_TXFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Tx failures" | awk '{ print $NF }') +ATH1_PCKERR=$(printf '%s\n' "$ATH1_RAW" | grep "Packets Errored" | awk '{ print $NF }') +ATH1_RETR=$(printf '%s\n' "$ATH1_RAW" | grep "Retries" | awk '{ print $NF }') +ATH1_BCNSUC=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon success" | awk '{ print $NF }') +ATH1_BCNFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon failed" | awk '{ print $NF }') + +ATH1_TXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') +ATH1_RXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') + + + +# ----------------------------------------------------------- +# channels +# ----------------------------------------------------------- + +ATH0_CH_NMBR=$(iw dev ath0 info | grep MHz | awk '{ print $2 }') +ATH0_CH_WIDTH=$(iw dev ath0 info | grep MHz | awk '{ print $6 }') +ATH0_CH_CNTR=$(iw dev ath0 info | grep MHz | awk '{ print $(NF-1) }') + +ATH1_CH_NMBR=$(iw dev ath1 info | grep MHz | awk '{ print $2 }') +ATH1_CH_WIDTH=$(iw dev ath1 info | grep MHz | awk '{ print $6 }') +ATH1_CH_CNTR=$(iw dev ath1 info | grep MHz | awk '{ print $(NF-1) }') + + +# --- ath1 RSSI buckets (gold/silver/bronze/trash) --- +ATH1_USGLD=0; ATH1_USSLV=0; ATH1_USBRN=0; ATH1_USTRS=0 + +_assigns="$( + ( + exec 2>/dev/null + hostapd_cli -i ath1 all_sta \ + | grep 'AUTHORIZED' -B1 -A12 \ + | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \ + | awk ' + BEGIN { g=0; s=0; b=0; t=0 } + /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next } + /^signal=/ { + sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0 + if (sig>=-65 && sig<=-20) g++ + else if (sig>=-75 && sig<=-66) s++ + else if (sig>=-85 && sig<=-76) b++ + else if (sig>=-200 && sig<=-86) t++ + } + END { + printf "ATH1_USGLD=%d\nATH1_USSLV=%d\nATH1_USBRN=%d\nATH1_USTRS=%d\n", g+0, s+0, b+0, t+0 + } + ' + ) || true +)" + +# apply only if well-formed; otherwise keep zeros +if printf '%s\n' "$_assigns" | awk -F= ' + NR==1 && $1=="ATH1_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1} + NR==2 && $1=="ATH1_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1} + NR==3 && $1=="ATH1_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1} + NR==4 && $1=="ATH1_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1} + END{ exit !(ok1&&ok2&&ok3&&ok4) } +'; then + eval "$_assigns" +fi + +# final sanitize +case "$ATH1_USGLD" in (''|*[!0-9]*) ATH1_USGLD=0;; esac +case "$ATH1_USSLV" in (''|*[!0-9]*) ATH1_USSLV=0;; esac +case "$ATH1_USBRN" in (''|*[!0-9]*) ATH1_USBRN=0;; esac +case "$ATH1_USTRS" in (''|*[!0-9]*) ATH1_USTRS=0;; esac + +######## + +# --- ath0 RSSI buckets (gold/silver/bronze/trash) --- +ATH0_USGLD=0; ATH0_USSLV=0; ATH0_USBRN=0; ATH0_USTRS=0 + +_assigns="$( + ( + exec 2>/dev/null + hostapd_cli -i ath0 all_sta \ + | grep 'AUTHORIZED' -B1 -A12 \ + | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \ + | awk ' + BEGIN { g=0; s=0; b=0; t=0 } + /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next } + /^signal=/ { + sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0 + if (sig>=-65 && sig<=-20) g++ + else if (sig>=-75 && sig<=-66) s++ + else if (sig>=-85 && sig<=-76) b++ + else if (sig>=-200 && sig<=-86) t++ + } + END { + printf "ATH0_USGLD=%d\nATH0_USSLV=%d\nATH0_USBRN=%d\nATH0_USTRS=%d\n", g+0, s+0, b+0, t+0 + } + ' + ) || true +)" +# apply only if well-formed; otherwise keep zeros +if printf '%s\n' "$_assigns" | awk -F= ' + NR==1 && $1=="ATH0_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1} + NR==2 && $1=="ATH0_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1} + NR==3 && $1=="ATH0_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1} + NR==4 && $1=="ATH0_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1} + END{ exit !(ok1&&ok2&&ok3&&ok4) } +'; then + eval "$_assigns" +fi + +# final sanitize +case "$ATH0_USGLD" in (''|*[!0-9]*) ATH0_USGLD=0;; esac +case "$ATH0_USSLV" in (''|*[!0-9]*) ATH0_USSLV=0;; esac +case "$ATH0_USBRN" in (''|*[!0-9]*) ATH0_USBRN=0;; esac +case "$ATH0_USTRS" in (''|*[!0-9]*) ATH0_USTRS=0;; esac + +# ----------------------------------------------------------- +# FCS totals/errored +# ----------------------------------------------------------- + +WIFISTATS_ATH0_FCSOK=$(wifistats wifi0 2 | grep fcs_ok | awk '{ print $3 }') +WIFISTATS_ATH0_FCSERR=$(wifistats wifi0 2 | grep fcs_err | awk '{ print $3 }') + +WIFISTATS_ATH1_FCSOK=$(wifistats wifi1 2 | grep fcs_ok | awk '{ print $3 }') +WIFISTATS_ATH1_FCSERR=$(wifistats wifi1 2 | grep fcs_err | awk '{ print $3 }') + + + +# ----------------------------------------------------------- +# Build message +# ----------------------------------------------------------- + +MSG="hostname=${HOSTNAME} \ +scrver=${SCRIPTVERSION} \ +ath1_ch_nmbr=${ATH1_CH_NMBR} \ +ath1_ch_width=${ATH1_CH_WIDTH} \ +ath1_ch_cntr=${ATH1_CH_CNTR} \ +ath1_obss_util=${ATH1_OBSS_UTIL} \ +ath1_chutil=${ATH1_CH_UTIL} \ +ath1_txdp=${ATH1_TXDP} \ +ath1_rxdp=${ATH1_RXDP} \ +ath1_txudp=${ATH1_TXUDP} \ +ath1_txmbdp=${ATH1_TXMBDP} \ +ath1_txmdp=${ATH1_TXMDP} \ +ath1_txbdp=${ATH1_TXBDP} \ +ath1_txflr=${ATH1_TXFLR} \ +ath1_txrate=${ATH1_TXRATE} \ +ath1_rxrate=${ATH1_RXRATE} \ +ath1_pckerr=${ATH1_PCKERR} \ +ath1_retr=${ATH1_RETR} \ +ath1_bcnsuc=${ATH1_BCNSUC} \ +ath1_bcnflr=${ATH1_BCNFLR} \ +ath1_usercount=${ATH1_USERCOUNT} \ +ath1_usgld=${ATH1_USGLD} \ +ath1_usslv=${ATH1_USSLV} \ +ath1_usbrn=${ATH1_USBRN} \ +ath1_ustrs=${ATH1_USTRS} \ +ath1_fcsok=${WIFISTATS_ATH1_FCSOK} \ +ath1_fcserr=${WIFISTATS_ATH1_FCSERR} \ +ath0_ch_nmbr=${ATH0_CH_NMBR} \ +ath0_ch_width=${ATH0_CH_WIDTH} \ +ath0_ch_cntr=${ATH0_CH_CNTR} \ +ath0_obss_util=${ATH0_OBSS_UTIL} \ +ath0_chutil=${ATH0_CH_UTIL} \ +ath0_txdp=${ATH0_TXDP} \ +ath0_rxdp=${ATH0_RXDP} \ +ath0_txudp=${ATH0_TXUDP} \ +ath0_txmbdp=${ATH0_TXMBDP} \ +ath0_txmdp=${ATH0_TXMDP} \ +ath0_txbdp=${ATH0_TXBDP} \ +ath0_txflr=${ATH0_TXFLR} \ +ath0_txrate=${ATH0_TXRATE} \ +ath0_rxrate=${ATH0_RXRATE} \ +ath0_pckerr=${ATH0_PCKERR} \ +ath0_retr=${ATH0_RETR} \ +ath0_bcnsuc=${ATH0_BCNSUC} \ +ath0_bcnflr=${ATH0_BCNFLR} \ +ath0_usercount=${ATH0_USERCOUNT} \ +ath0_usgld=${ATH0_USGLD} \ +ath0_usslv=${ATH0_USSLV} \ +ath0_usbrn=${ATH0_USBRN} \ +ath0_ustrs=${ATH0_USTRS} \ +ath0_fcsok=${WIFISTATS_ATH0_FCSOK} \ +ath0_fcserr=${WIFISTATS_ATH0_FCSERR} \ +fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW}" + + + +# Send message via syslog +logger -t "debug|${HOSTNAME}" "$MSG" + diff --git a/files/files/wifidebug16.sh b/files/files/wifidebug16.sh new file mode 100644 index 0000000..552625c --- /dev/null +++ b/files/files/wifidebug16.sh @@ -0,0 +1,420 @@ +#!/bin/sh + +#sleep $((RANDOM % 18 + 3)) + +SCRIPTVERSION=16 + +delay=$(( ( $(hexdump -n2 -e '/2 "%u"' /dev/urandom) % 18 ) + 3 )) +#echo "Sleeping for $delay seconds..." +sleep $delay + +# Get hostname +HOSTNAME=$(cat /proc/sys/kernel/hostname) + +FWVER=$(cat /etc/banner | grep rev | awk '{ print $1 }') + +# =========================================================== +# GPS coordinates management (locks, timeout, 0/0 fallback) +# - Writes /tmp/gps_coordinates with: +# longitude: +# latitude: +# - Uses /tmp/gps_coordinates..lock (expires 10m) +# - Tracks retries in /tmp/gps_timeout (stop after 3) +# - On any retrieval attempt (success/fail) -> exit cycle +# =========================================================== + +COORD_FILE="/tmp/gps_coordinates" +TIMEOUT_FILE="/tmp/gps_timeout" +LOCK_MAX_AGE_MIN=10 # minutes +RETRIEVE_MAX_WAIT=480 # seconds (8 minutes) +LOCATION_LON="0.000000" +LOCATION_LAT="0.000000" + +# Helper: parse existing coords file (if any) +read_coords_file() { + if [ -f "$COORD_FILE" ]; then + # Expect two lines: "longitude: X" and "latitude: Y" + local lon lat + lon=$(awk -F': *' '/^longitude:/ {print $2; exit}' "$COORD_FILE") + lat=$(awk -F': *' '/^latitude:/ {print $2; exit}' "$COORD_FILE") + # Validate numeric and ranges; format to 6 decimals + if echo "$lon" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then + if echo "$lat" | awk 'BEGIN{ok=0} /^[+-]?[0-9]*\.?[0-9]+$/ {ok=1} END{exit !ok}'; then + if awk -v a="$lat" -v b="$lon" 'BEGIN{exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}'; then + LOCATION_LON=$(awk -v x="$lon" 'BEGIN{printf "%.6f", x+0}') + LOCATION_LAT=$(awk -v x="$lat" 'BEGIN{printf "%.6f", x+0}') + return 0 + fi + fi + fi + fi + return 1 +} + +# If coords exist and valid -> proceed; else attempt retrieval logic +if read_coords_file; then + : +else + # Check retry threshold + GTO=0 + if [ -f "$TIMEOUT_FILE" ]; then + GTO=$(cat "$TIMEOUT_FILE" 2>/dev/null | awk '/^[0-9]+$/ {print $0; exit}') + [ -z "$GTO" ] && GTO=0 + fi + + if [ "$GTO" -ge 3 ]; then + # Give up trying; ensure coords file exists with 0/0 and exit this cycle + printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp" + mv "${COORD_FILE}.tmp" "$COORD_FILE" + exit 0 + fi + + # Clean up stale locks (>10 minutes) + find /tmp -maxdepth 1 -type f -name 'gps_coordinates.*.lock' -mmin +"$LOCK_MAX_AGE_MIN" -exec rm -f {} \; 2>/dev/null + + # If any non-stale lock remains, skip this run + for f in /tmp/gps_coordinates.*.lock; do + [ -e "$f" ] || continue + # Found an active (<=10m) lock; skip this cycle + exit 0 + done + + # Create fresh lock and start retrieval + NOWTS=$(date +%s) + LOCK_FILE="/tmp/gps_coordinates.${NOWTS}.lock" + : > "$LOCK_FILE" 2>/dev/null + + CAND="/tmp/gps_candidate.$$" + ( + gpspipe -r | awk -F, ' + # -------- THE ONLY CHANGE IS HERE: use deglen+1 for minutes -------- + function dm2dd(dm, hemi, deglen, deg, min, dd) { + if (dm=="" || hemi=="") return "" + deglen = (hemi=="N" || hemi=="S") ? 2 : 3 + deg = substr(dm, 1, deglen) + 0 + min = substr(dm, deglen+1) + 0 + dd = deg + (min / 60.0) + return (hemi=="S" || hemi=="W") ? -dd : dd + } + # Wait for first valid RMC (A) + /^\$..RMC/ && $3=="A" { + lat = dm2dd($4, $5) + lon = dm2dd($6, $7) + if (lat != "" && lon != "") { + printf("%.6f %.6f\n", lon, lat) + exit + } + } + ' > "$CAND" + ) & + GPID=$! + + # Wait up to RETRIEVE_MAX_WAIT seconds for candidate to appear + waited=0 + while [ "$waited" -lt "$RETRIEVE_MAX_WAIT" ]; do + if [ -s "$CAND" ]; then + break + fi + sleep 1 + waited=$((waited+1)) + done + + success=0 + if [ -s "$CAND" ]; then + set -- $(head -n1 "$CAND" 2>/dev/null) + cand_lon="$1" + cand_lat="$2" + if awk -v a="$cand_lat" -v b="$cand_lon" 'BEGIN{ok=(a!="" && b!=""); if (!ok) exit 1; exit !(a>=-90 && a<=90 && b>=-180 && b<=180)}' + then + printf "longitude: %.6f\nlatitude: %.6f\n" "$cand_lon" "$cand_lat" > "${COORD_FILE}.tmp" + mv "${COORD_FILE}.tmp" "$COORD_FILE" + echo 0 > "$TIMEOUT_FILE" + success=1 + fi + fi + + # Cleanup background gpspipe if still running + kill "$GPID" 2>/dev/null + sleep 1 + kill -9 "$GPID" 2>/dev/null + + # Remove lock and temp + rm -f "$LOCK_FILE" "$CAND" 2>/dev/null + + # Exit this cycle after any retrieval attempt (success or fail) + if [ "$success" -eq 1 ]; then + exit 0 + else + printf "longitude: %.6f\nlatitude: %.6f\n" 0 0 > "${COORD_FILE}.tmp" + mv "${COORD_FILE}.tmp" "$COORD_FILE" + GTO=$((GTO+1)) + echo "$GTO" > "$TIMEOUT_FILE" + exit 0 + fi +fi + +# At this point, LOCATION_LON/LOCATION_LAT are set (from file or default 0/0) + +# Get OBSS utilization value from wifitool +ATH0_OBSS_UTIL=$(wifitool ath0 get_rrmutil | awk '/obss util/ { print $4 }') +ATH1_OBSS_UTIL=$(wifitool ath1 get_rrmutil | awk '/obss util/ { print $4 }') +ATH0_CH_UTIL=$(cfg80211tool ath0 get_chutil | awk -F ':' '{ print $2 }') +ATH1_CH_UTIL=$(cfg80211tool ath1 get_chutil | awk -F ':' '{ print $2 }') + +# ----------------------------------------------------------- +# Count connected (real) clients on ath0 and ath1 +# ----------------------------------------------------------- + +get_usercount() { + IFACE=$1 + hostapd_cli -i "$IFACE" all_sta 2>/dev/null | awk ' + /^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/ {mac=$0; next} + /flags=/ { + auth = (index($0, "AUTH") > 0) + assoc = (index($0, "ASSOC") > 0) + authorized = (index($0, "AUTHORIZED") > 0) + } + /rx_packets=/ {rx_packets=$1; sub("rx_packets=", "", rx_packets)} + /tx_packets=/ {tx_packets=$1; sub("tx_packets=", "", tx_packets)} + /inactive_msec=/ {inactive=$1; sub("inactive_msec=", "", inactive)} + /connected_time=/ { + connected=$1; sub("connected_time=", "", connected) + if (auth && assoc && authorized && + rx_packets+0 > 0 && tx_packets+0 > 0 && + inactive+0 < 5000 && connected+0 > 60) { + count++ + } + } + END { print count+0 } + ' +} + +ATH0_USERCOUNT=$(get_usercount ath0) +ATH1_USERCOUNT=$(get_usercount ath1) + +# ----------------------------------------------------------- +# Get IP/MAC for br-wan +# ----------------------------------------------------------- + +MACADDRESS=$(ip link show br-wan | awk '/link\/ether/ {print $2}') +IPADDR_MASK=$(ip -4 addr show br-wan | awk '/inet / {print $2}' | head -n 1) +IPADDRESS=${IPADDR_MASK%/*} +IPMASK=${IPADDR_MASK#*/} + +# ----------------------------------------------------------- +# Get LLDP neighbor switch +NEIGH_SW=$(grep ikeja /tmp/run/lldp_server.json | grep -E "\-sc|\-as" -m1 | sed -E 's/.*"system_name": "([^"]+)".*/\1/') + +# ----------------------------------------------------------- +# Get default GW +DEFGW=$(ip route show | grep default | awk '{ print $3 }') + +# ----------------------------------------------------------- + +# Run once per iface and reuse the buffer +ATH0_RAW="$(apstats -v -i ath0 2>/dev/null)" +ATH1_RAW="$(apstats -v -i ath1 2>/dev/null)" + +# ---- ath0 (read from ATH0_RAW) ---- +ATH0_TXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }') +ATH0_RXDP=$(printf '%s\n' "$ATH0_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }') +ATH0_TXUDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }') +ATH0_TXMBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }') +ATH0_TXMDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }') +ATH0_TXBDP=$(printf '%s\n' "$ATH0_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }') +ATH0_TXFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Tx failures" | awk '{ print $NF }') +ATH0_PCKERR=$(printf '%s\n' "$ATH0_RAW" | grep "Packets Errored" | awk '{ print $NF }') +ATH0_RETR=$(printf '%s\n' "$ATH0_RAW" | grep "Retries" | awk '{ print $NF }') +ATH0_BCNSUC=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon success" | awk '{ print $NF }') +ATH0_BCNFLR=$(printf '%s\n' "$ATH0_RAW" | grep "Beacon failed" | awk '{ print $NF }') + +ATH0_TXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') +ATH0_RXRATE=$(echo "$ATH0_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') + +# ---- ath1 (read from ATH1_RAW) ---- +ATH1_TXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Tx Data Packets " | awk '{ print $NF }') +ATH1_RXDP=$(printf '%s\n' "$ATH1_RAW" | grep -m 1 "Rx Data Packets" | awk '{ print $NF }') +ATH1_TXUDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Unicast Data Packets" | awk '{ print $NF }') +ATH1_TXMBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Multi/Broadcast Data Packets" | awk '{ print $NF }') +ATH1_TXMDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx multicast Data Packets" | awk '{ print $NF }') +ATH1_TXBDP=$(printf '%s\n' "$ATH1_RAW" | grep "Tx Broadcast Data Packets" | awk '{ print $NF }') +ATH1_TXFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Tx failures" | awk '{ print $NF }') +ATH1_PCKERR=$(printf '%s\n' "$ATH1_RAW" | grep "Packets Errored" | awk '{ print $NF }') +ATH1_RETR=$(printf '%s\n' "$ATH1_RAW" | grep "Retries" | awk '{ print $NF }') +ATH1_BCNSUC=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon success" | awk '{ print $NF }') +ATH1_BCNFLR=$(printf '%s\n' "$ATH1_RAW" | grep "Beacon failed" | awk '{ print $NF }') + +ATH1_TXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Tx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') +ATH1_RXRATE=$(echo "$ATH1_RAW" | awk -F'=' '/^Average Rx Rate/{gsub(/[^0-9]/,"",$2); print ($2==""?0:$2); exit}') + +# ----------------------------------------------------------- +# channels +# ----------------------------------------------------------- + +ATH0_CH_NMBR=$(iw dev ath0 info | grep MHz | awk '{ print $2 }') +ATH0_CH_WIDTH=$(iw dev ath0 info | grep MHz | awk '{ print $6 }') +ATH0_CH_CNTR=$(iw dev ath0 info | grep MHz | awk '{ print $(NF-1) }') + +ATH1_CH_NMBR=$(iw dev ath1 info | grep MHz | awk '{ print $2 }') +ATH1_CH_WIDTH=$(iw dev ath1 info | grep MHz | awk '{ print $6 }') +ATH1_CH_CNTR=$(iw dev ath1 info | grep MHz | awk '{ print $(NF-1) }') + +# --- ath1 RSSI buckets (gold/silver/bronze/trash) --- +ATH1_USGLD=0; ATH1_USSLV=0; ATH1_USBRN=0; ATH1_USTRS=0 +_assigns="$( + ( + exec 2>/dev/null + hostapd_cli -i ath1 all_sta \ + | grep 'AUTHORIZED' -B1 -A12 \ + | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \ + | awk ' + BEGIN { g=0; s=0; b=0; t=0 } + /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next } + /^signal=/ { + sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0 + if (sig>=-65 && sig<=-20) g++ + else if (sig>=-75 && sig<=-66) s++ + else if (sig>=-85 && sig<=-76) b++ + else if (sig>=-200 && sig<=-86) t++ + } + END { + printf "ATH1_USGLD=%d\nATH1_USSLV=%d\nATH1_USBRN=%d\nATH1_USTRS=%d\n", g+0, s+0, b+0, t+0 + } + ' + ) || true +)" +if printf '%s\n' "$_assigns" | awk -F= ' + NR==1 && $1=="ATH1_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1} + NR==2 && $1=="ATH1_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1} + NR==3 && $1=="ATH1_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1} + NR==4 && $1=="ATH1_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1} + END{ exit !(ok1&&ok2&&ok3&&ok4) } +'; then + eval "$_assigns" +fi +case "$ATH1_USGLD" in (''|*[!0-9]*) ATH1_USGLD=0;; esac +case "$ATH1_USSLV" in (''|*[!0-9]*) ATH1_USSLV=0;; esac +case "$ATH1_USBRN" in (''|*[!0-9]*) ATH1_USBRN=0;; esac +case "$ATH1_USTRS" in (''|*[!0-9]*) ATH1_USTRS=0;; esac + +# --- ath0 RSSI buckets (gold/silver/bronze/trash) --- +ATH0_USGLD=0; ATH0_USSLV=0; ATH0_USBRN=0; ATH0_USTRS=0 +_assigns="$( + ( + exec 2>/dev/null + hostapd_cli -i ath0 all_sta \ + | grep 'AUTHORIZED' -B1 -A12 \ + | grep -E '([[:xdigit:]]{2}:){5}[[:xdigit:]]{2}|^signal=' \ + | awk ' + BEGIN { g=0; s=0; b=0; t=0 } + /^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$/ { next } + /^signal=/ { + sig=$0; sub(/^signal=[[:space:]]*/,"",sig); sig+=0 + if (sig>=-65 && sig<=-20) g++ + else if (sig>=-75 && sig<=-66) s++ + else if (sig>=-85 && sig<=-76) b++ + else if (sig>=-200 && sig<=-86) t++ + } + END { + printf "ATH0_USGLD=%d\nATH0_USSLV=%d\nATH0_USBRN=%d\nATH0_USTRS=%d\n", g+0, s+0, b+0, t+0 + } + ' + ) || true +)" +if printf '%s\n' "$_assigns" | awk -F= ' + NR==1 && $1=="ATH0_USGLD" && $2 ~ /^[0-9]+$/ {ok1=1} + NR==2 && $1=="ATH0_USSLV" && $2 ~ /^[0-9]+$/ {ok2=1} + NR==3 && $1=="ATH0_USBRN" && $2 ~ /^[0-9]+$/ {ok3=1} + NR==4 && $1=="ATH0_USTRS" && $2 ~ /^[0-9]+$/ {ok4=1} + END{ exit !(ok1&&ok2&&ok3&&ok4) } +'; then + eval "$_assigns" +fi +case "$ATH0_USGLD" in (''|*[!0-9]*) ATH0_USGLD=0;; esac +case "$ATH0_USSLV" in (''|*[!0-9]*) ATH0_USSLV=0;; esac +case "$ATH0_USBRN" in (''|*[!0-9]*) ATH0_USBRN=0;; esac +case "$ATH0_USTRS" in (''|*[!0-9]*) ATH0_USTRS=0;; esac + +# ----------------------------------------------------------- +# FCS totals/errored +# ----------------------------------------------------------- + +WIFISTATS_ATH0_FCSOK=$(wifistats wifi0 2 | grep fcs_ok | awk '{ print $3 }') +WIFISTATS_ATH0_FCSERR=$(wifistats wifi0 2 | grep fcs_err | awk '{ print $3 }') + +WIFISTATS_ATH1_FCSOK=$(wifistats wifi1 2 | grep fcs_ok | awk '{ print $3 }') +WIFISTATS_ATH1_FCSERR=$(wifistats wifi1 2 | grep fcs_err | awk '{ print $3 }') + +# 10-hex-char uuid for correlation (shared between the two parts) +UUID=$(hexdump -n5 -e '5/1 "%02x"' /dev/urandom) + +# ------------------------ +# Part 1: ath0 + shared (+ GPS) +# ------------------------ +MSG1="hostname=${HOSTNAME} \ +uuid=${UUID} part=1 \ +scrver=${SCRIPTVERSION} \ +ath0_ch_nmbr=${ATH0_CH_NMBR} \ +ath0_ch_width=${ATH0_CH_WIDTH} \ +ath0_ch_cntr=${ATH0_CH_CNTR} \ +ath0_obss_util=${ATH0_OBSS_UTIL} \ +ath0_chutil=${ATH0_CH_UTIL} \ +ath0_txdp=${ATH0_TXDP} \ +ath0_rxdp=${ATH0_RXDP} \ +ath0_txudp=${ATH0_TXUDP} \ +ath0_txmbdp=${ATH0_TXMBDP} \ +ath0_txmdp=${ATH0_TXMDP} \ +ath0_txbdp=${ATH0_TXBDP} \ +ath0_txflr=${ATH0_TXFLR} \ +ath0_txrate=${ATH0_TXRATE} \ +ath0_rxrate=${ATH0_RXRATE} \ +ath0_pckerr=${ATH0_PCKERR} \ +ath0_retr=${ATH0_RETR} \ +ath0_bcnsuc=${ATH0_BCNSUC} \ +ath0_bcnflr=${ATH0_BCNFLR} \ +ath0_usercount=${ATH0_USERCOUNT} \ +ath0_usgld=${ATH0_USGLD} \ +ath0_usslv=${ATH0_USSLV} \ +ath0_usbrn=${ATH0_USBRN} \ +ath0_ustrs=${ATH0_USTRS} \ +ath0_fcsok=${WIFISTATS_ATH0_FCSOK} \ +ath0_fcserr=${WIFISTATS_ATH0_FCSERR} \ +fw_ver=${FWVER} ipaddress=${IPADDRESS} ipmask=${IPMASK} defgw=${DEFGW} macaddress=${MACADDRESS} neigh_switch=${NEIGH_SW} \ +location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}" + +# ------------------------ +# Part 2: ath1 only +# ------------------------ +MSG2="hostname=${HOSTNAME} \ +uuid=${UUID} part=2 \ +scrver=${SCRIPTVERSION} \ +ath1_ch_nmbr=${ATH1_CH_NMBR} \ +ath1_ch_width=${ATH1_CH_WIDTH} \ +ath1_ch_cntr=${ATH1_CH_CNTR} \ +ath1_obss_util=${ATH1_OBSS_UTIL} \ +ath1_chutil=${ATH1_CH_UTIL} \ +ath1_txdp=${ATH1_TXDP} \ +ath1_rxdp=${ATH1_RXDP} \ +ath1_txudp=${ATH1_TXUDP} \ +ath1_txmbdp=${ATH1_TXMBDP} \ +ath1_txmdp=${ATH1_TXMDP} \ +ath1_txbdp=${ATH1_TXBDP} \ +ath1_txflr=${ATH1_TXFLR} \ +ath1_txrate=${ATH1_TXRATE} \ +ath1_rxrate=${ATH1_RXRATE} \ +ath1_pckerr=${ATH1_PCKERR} \ +ath1_retr=${ATH1_RETR} \ +ath1_bcnsuc=${ATH1_BCNSUC} \ +ath1_bcnflr=${ATH1_BCNFLR} \ +ath1_usercount=${ATH1_USERCOUNT} \ +ath1_usgld=${ATH1_USGLD} \ +ath1_usslv=${ATH1_USSLV} \ +ath1_usbrn=${ATH1_USBRN} \ +ath1_ustrs=${ATH1_USTRS} \ +ath1_fcsok=${WIFISTATS_ATH1_FCSOK} \ +ath1_fcserr=${WIFISTATS_ATH1_FCSERR} \ +location.lat=${LOCATION_LAT} location.lon=${LOCATION_LON}" + +# Send messages via syslog (two separate lines, same tag) +logger -t "debug|${HOSTNAME}" "$MSG1" +logger -t "debug|${HOSTNAME}" "$MSG2" diff --git a/files/files/wirelessduediligence.sh b/files/files/wirelessduediligence.sh new file mode 100644 index 0000000..bdf36e1 --- /dev/null +++ b/files/files/wirelessduediligence.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# Usage: ./duediligence.sh [/path/to/config.json] +set -eu + +CFG="${1:-/tmp/config.json}" + +if ! command -v jq >/dev/null 2>&1; then + echo '{"error":"jq not found in PATH"}' + exit 2 +fi + +if [ ! -f "$CFG" ]; then + # Still emit JSON so Ansible can parse the error + printf '{"error":"config not found: %s"}\n' "$CFG" + exit 2 +fi + +jq -c ' + def classify(n): + if n==0 then "NO_VAPS" + elif n==1 then "ONE_VAP" + elif n==2 then "TWO_VAPS" + else "MORE_THAN_TWO_VAPS" end; + + def radio_summary($r): + ( .wireless.radios[$r] // {} ) as $rad + | ( $rad.vaps // [] ) as $v + | { + radio_enabled: ($rad.enabled // false), + radio_mode: ($rad.mode // "unknown"), + vaps_total: ($v | length), + indices: ( $v | to_entries | map(.key) ), + enabled_ap_indices: + ( $v + | to_entries + | map(select(.value.mode=="ap" and (.value.enabled==true)) | .key) + ), + classification: classify($v | length), + vaps: + ( $v + | to_entries + | map({ + index: .key, + mode: (.value.mode // "null"), + enabled: (.value.enabled // false), + lbd: (.value.lbd // null), + zone: (.value.network.zone // null), + ssid: (.value.ssid // "") + }) + ) + }; + + { + file: input_filename, + radios: { + wifi0: radio_summary("wifi0"), + wifi1: radio_summary("wifi1"), + wlan0: radio_summary("wlan0") + }, + global: { + preferred5G: (.wireless.lbd.preferred5G // null) + } + } +' "$CFG" diff --git a/files/fox200-2.2.1-r6801.bin b/files/fox200-2.2.1-r6801.bin new file mode 100644 index 0000000..80ece61 Binary files /dev/null and b/files/fox200-2.2.1-r6801.bin differ diff --git a/files/group_vars/all.yml b/files/group_vars/all.yml new file mode 100644 index 0000000..fa36561 --- /dev/null +++ b/files/group_vars/all.yml @@ -0,0 +1,5 @@ +ansible_user: root # or the user your devices accept +ansible_port: 22 +host_key_checking: false # optional: you already have this in ansible.cfg +# ansible_ssh_private_key_file: ~/.ssh/id_rsa # if you use a key + diff --git a/files/inventory/netbox.yml b/files/inventory/netbox.yml new file mode 100644 index 0000000..c20af8f --- /dev/null +++ b/files/inventory/netbox.yml @@ -0,0 +1,15 @@ +# inventory/netbox.yml +plugin: netbox.netbox.nb_inventory + +api_endpoint: "http://netbox.gt-tiso.ikeja.co.za" # <-- no trailing /api here +token: "7648e4f5ee370cda7834682e61b47c2ee8e95623" + +query_filters: + - status: active + - has_primary_ip: True + +compose: + ansible_host: >- + {{ (primary_ip4.address | default('')) | regex_replace('/\\d+$', '') }} + + diff --git a/files/nbplay b/files/nbplay new file mode 100755 index 0000000..97396a8 --- /dev/null +++ b/files/nbplay @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Hardcoded NetBox access (DEV ONLY!) +NETBOX_URL="http://netbox.gt-tiso.ikeja.co.za" +NETBOX_TOKEN="7648e4f5ee370cda7834682e61b47c2ee8e95623" + +usage() { + cat >&2 < [ansible-playbook args...] + --pwfile PATH Path to password/vars file: + - YAML/JSON: must contain ansible_ssh_pass (and optionally ansible_user) + - Plain text: first line is the password (wrapped as ansible_ssh_pass) + --user USER ansible_user to use (overrides file value unless omitted) + --rebootin VAL Passes playbook var rebootin=VAL (e.g. "23:35" or "120") +Examples: + nbplay update-indoor.yml ikeja12345 --pwfile /opt/.../ssh.yml --rebootin "23:35" + nbplay update-indoor.yml ikeja12345 --pwfile /opt/.../ssh.txt --user root --rebootin "120" +EOF + exit 2 +} + +# ---- Parse args +PWFILE="" +USER_OVERRIDE="" +REBOOTIN_VALUE="" +ARGS=() +while (( $# )); do + case "${1:-}" in + --pwfile) shift; PWFILE="${1:-}"; [[ -n "$PWFILE" ]] || usage; shift;; + --user) shift; USER_OVERRIDE="${1:-}"; [[ -n "$USER_OVERRIDE" ]] || usage; shift;; + --rebootin) shift; REBOOTIN_VALUE="${1:-}"; [[ -n "$REBOOTIN_VALUE" ]] || usage; shift;; + -h|--help) usage;; + *) ARGS+=("$1"); shift;; + esac +done +set -- "${ARGS[@]}" + +if [[ $# -lt 2 ]]; then usage; fi +PLAYBOOK="$1"; shift +NAME="$1"; shift + +if [[ -z "${PWFILE}" ]]; then + DEFAULT_PWFILE="${NBPLAY_PWFILE:-/opt/containers/ansible-worker/data/ssh.yml}" + if [[ -f "$DEFAULT_PWFILE" ]]; then + PWFILE="$DEFAULT_PWFILE" + fi +fi + +# ---- Deps +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 127; } +command -v curl >/dev/null 2>&1 || { echo "curl not found" >&2; exit 127; } + +BASE="${NETBOX_URL%/}" +hdr=(-H "Authorization: Token ${NETBOX_TOKEN}" -H "Accept: application/json") + +# ---- Resolve device -> IP +dev_json="$(curl -fsS "${hdr[@]}" "${BASE}/api/dcim/devices/?name=${NAME}&limit=1")" +count="$(printf '%s' "$dev_json" | jq -r '.count // 0')" +if [[ "$count" != "1" ]]; then + echo "Device '${NAME}' not found or not unique (count=${count})." >&2 + exit 1 +fi +addr_v4="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip4.address // empty')" +addr_v6="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip6.address // empty')" +addr_any="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip.address // empty')" +addr="${addr_v4:-${addr_v6:-${addr_any:-}}}" +[[ -n "$addr" && "$addr" != "null" ]] || { echo "Device '${NAME}' has no primary IP." >&2; exit 1; } +ip="${addr%%/*}" + +# ---- Temp inventory +tmpinv="$(mktemp)"; trap 'rm -f "$tmpinv" "$tmpvars"' EXIT +cat >"$tmpinv" <&2; exit 1; } + pass="$(head -n1 "$PWFILE" | tr -d '\r\n')" + ensure_tmpvars + { + echo "ansible_ssh_pass: \"$pass\"" + [[ -n "$USER_OVERRIDE" ]] && echo "ansible_user: \"$USER_OVERRIDE\"" + } >> "$tmpvars" + fi +fi + +# If user override provided and not already set via tmpvars/YAML, add it safely +if [[ -n "$USER_OVERRIDE" && -z "$tmpvars" ]]; then + ensure_tmpvars + echo "ansible_user: \"$USER_OVERRIDE\"" >> "$tmpvars" +fi + +# If rebootin provided, add it to the temp vars file +if [[ -n "$REBOOTIN_VALUE" ]]; then + ensure_tmpvars + echo "rebootin: \"$REBOOTIN_VALUE\"" >> "$tmpvars" +fi + +# ---- Run playbook (do NOT use --ask-pass) +ANSIBLE_ASK_PASS=False exec ansible-playbook -i "$tmpinv" "$PLAYBOOK" -l "$NAME" \ + "${extravars[@]}" "$@" diff --git a/files/nbplay_beforeIndoors b/files/nbplay_beforeIndoors new file mode 100755 index 0000000..ae13b76 --- /dev/null +++ b/files/nbplay_beforeIndoors @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Hardcoded NetBox access (DEV ONLY!) +NETBOX_URL="http://netbox.gt-tiso.ikeja.co.za" +NETBOX_TOKEN="7648e4f5ee370cda7834682e61b47c2ee8e95623" + +usage() { + cat >&2 < [ansible-playbook args...] + --pwfile PATH Path to password/vars file: + - YAML/JSON: must contain ansible_ssh_pass (and optionally ansible_user) + - Plain text: first line is the password (wrapped as ansible_ssh_pass) + --user USER ansible_user to use (overrides file value unless omitted) +Examples: + nbplay uptime.yml ikeja12345 --pwfile /opt/containers/ansible-worker/data/ssh.yml + nbplay uptime.yml ikeja12345 --pwfile /opt/.../ssh.txt --user root +EOF + exit 2 +} + +# ---- Parse args +PWFILE="" +USER_OVERRIDE="" +ARGS=() +while (( $# )); do + case "${1:-}" in + --pwfile) shift; PWFILE="${1:-}"; [[ -n "$PWFILE" ]] || usage; shift;; + --user) shift; USER_OVERRIDE="${1:-}"; [[ -n "$USER_OVERRIDE" ]] || usage; shift;; + -h|--help) usage;; + *) ARGS+=("$1"); shift;; + esac +done +set -- "${ARGS[@]}" + +if [[ $# -lt 2 ]]; then usage; fi +PLAYBOOK="$1"; shift +NAME="$1"; shift + +if [[ -z "${PWFILE}" ]]; then + DEFAULT_PWFILE="${NBPLAY_PWFILE:-/opt/containers/ansible-worker/data/ssh.yml}" + if [[ -f "$DEFAULT_PWFILE" ]]; then + PWFILE="$DEFAULT_PWFILE" + fi +fi + +# ---- Deps +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 127; } +command -v curl >/dev/null 2>&1 || { echo "curl not found" >&2; exit 127; } + +BASE="${NETBOX_URL%/}" +hdr=(-H "Authorization: Token ${NETBOX_TOKEN}" -H "Accept: application/json") + +# ---- Resolve device -> IP +dev_json="$(curl -fsS "${hdr[@]}" "${BASE}/api/dcim/devices/?name=${NAME}&limit=1")" +count="$(printf '%s' "$dev_json" | jq -r '.count // 0')" +if [[ "$count" != "1" ]]; then + echo "Device '${NAME}' not found or not unique (count=${count})." >&2 + exit 1 +fi +addr_v4="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip4.address // empty')" +addr_v6="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip6.address // empty')" +addr_any="$(printf '%s' "$dev_json" | jq -r '.results[0].primary_ip.address // empty')" +addr="${addr_v4:-${addr_v6:-${addr_any:-}}}" +[[ -n "$addr" && "$addr" != "null" ]] || { echo "Device '${NAME}' has no primary IP." >&2; exit 1; } +ip="${addr%%/*}" + +# ---- Temp inventory +tmpinv="$(mktemp)"; trap 'rm -f "$tmpinv" "$tmpvars"' EXIT +cat >"$tmpinv" <&2; exit 1; } + pass="$(head -n1 "$PWFILE" | tr -d '\r\n')" + tmpvars="$(mktemp)" + { + echo "ansible_ssh_pass: \"$pass\"" + [[ -n "$USER_OVERRIDE" ]] && echo "ansible_user: \"$USER_OVERRIDE\"" + } > "$tmpvars" + chmod 600 "$tmpvars" + extravars+=(-e "@${tmpvars}") + fi +fi + +# If user override provided and not already set via tmpvars/YAML, add it safely +if [[ -n "$USER_OVERRIDE" && -z "$tmpvars" ]]; then + tmpvars="$(mktemp)" + echo "ansible_user: \"$USER_OVERRIDE\"" > "$tmpvars" + chmod 600 "$tmpvars" + extravars+=(-e "@${tmpvars}") +fi + +# ---- Run playbook (do NOT use --ask-pass) +ANSIBLE_ASK_PASS=False exec ansible-playbook -i "$tmpinv" "$PLAYBOOK" -l "$NAME" \ + "${extravars[@]}" "$@" diff --git a/files/rabbit-client-old.sh b/files/rabbit-client-old.sh new file mode 100644 index 0000000..c366c8a --- /dev/null +++ b/files/rabbit-client-old.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env or flags) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Mode A: read from a queue (default) +QUEUE="${QUEUE:-queue1}" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # e.g. "amq.topic" or "my-exchange". Leave empty to skip binding mode. +ROUTING_KEY="${ROUTING_KEY:-#}" # pattern for topic/direct, default is catch-all "#" + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Pretty-print with jq if available +USE_JQ="${USE_JQ:-auto}" # auto|yes|no +USE_COLOR="${USE_COLOR:-yes}" # yes|no -> 'yes' forces color with jq -C + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +pretty() { + if [[ "$USE_JQ" == "yes" ]] || { [[ "$USE_JQ" == "auto" ]] && has_jq; }; then + if [[ "$USE_COLOR" == "yes" ]]; then jq -C .; else jq .; fi + else + cat + fi +} + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Map task_name -> absolute playbook path +map_playbook() { + case "$1" in + uptime) printf '%s/uptime.yml' "${APP_ROOT}" ;; + reset_aths) printf '%s/resetradios.yml' "${APP_ROOT}" ;; + *) return 1 ;; + esac +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task delay playbook + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + delay="$(jq -r '.task_delay // empty' <<<"$json")" || delay="" + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + if ! playbook="$(map_playbook "$task")"; then + warn "unknown task_name='$task' for device='$device' (no-op)." + return 0 + fi + + # Optional delay: "none" -> no delay; integer -> sleep seconds + if [[ -n "$delay" && "$delay" != "none" ]]; then + if [[ "$delay" =~ ^[0-9]+$ ]]; then + log "delaying ${delay}s before running '${task}' on '${device}'" + sleep "$delay" + else + warn "task_delay value '$delay' not numeric/'none' (ignoring)." + fi + fi + + log "→ Running playbook $(basename "$playbook") on host '${device}'" + if ! "${NBPLAY}" "$playbook" "$device"; then + err "playbook failed (task='${task}', device='${device}')" + # do not exit; keep consuming + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide queue source --- +if [[ -n "$EXCHANGE" ]]; then + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" +else + log "Consuming directly from queue: $QUEUE (vhost: $VHOST)" +fi + +log "Press Ctrl+C to stop." + +# --- Consume loop --- +while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + # Empty array => no messages + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + # Process one-by-one (the API returns an array) + if has_jq && { [[ "$USE_JQ" == "yes" ]] || [[ "$USE_JQ" == "auto" ]]; }; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + rk=$(printf '%s' "$item" | jq -r '.routing_key') + ex=$(printf '%s' "$item" | jq -r '.exchange') + + echo "-----" + echo "exchange: ${ex:-\"\"}" + echo "routing_key: $rk" + echo "payload:" + + # Decode + pretty print + DISPATCH + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + printf '%s' "$decoded" | pretty + dispatch_task "$decoded" + else + printf '%s\n' "$decoded" + # not JSON -> no dispatch + fi + else + printf '%s' "$payload" | pretty + dispatch_task "$payload" + fi + else + printf '%s\n' "$payload" + # not JSON -> no dispatch + fi + done + else + # Minimal fallback: print raw response only + echo "$RESP" + fi +done diff --git a/files/rabbit-client.sh b/files/rabbit-client.sh new file mode 100644 index 0000000..29178e6 --- /dev/null +++ b/files/rabbit-client.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Hard override as in your current file +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default). Supports CSV for multiple queues. +# CHANGED: default now includes both normal and persuasive work queues. +QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}" # e.g. "queue_deviceconfig,queue_persuasive" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth) + if [[ "$task" == "afterupgrade_check" ]]; then + local attempt corr_id orig_at target_ver + attempt="$(jq -r '.attempt // empty' <<<"$json")" + corr_id="$(jq -r '.correlation_id // empty' <<<"$json")" + orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")" + target_ver="$(jq -r '.target_version // empty' <<<"$json")" + + [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}" + [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}" + [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}" + + if [[ -n "$target_ver" ]]; then + local esc_tv=${target_ver//\'/\'\"\'\"\'} # replace ' with '\'' safely + task_options+=" -e target_version='${esc_tv}'" + fi + fi + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special cases: parametric reboot hours from task name + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + + # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4 + if [[ "$task" =~ ^(.+)_tonight$ ]]; then + base_task="${BASH_REMATCH[1]}" + + # now, today 01:00, tomorrow 01:00 (local time) + local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h + now_s="$(date +%s)" + today1_s="$(date -d 'today 01:00' +%s)" + tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)" + if (( now_s < today1_s )); then + next1_s="$today1_s" + else + next1_s="$tomorrow1_s" + fi + diff_s=$(( next1_s - now_s )) + # Ceil hours so current minutes are preserved as in your examples + ceil_h=$(( (diff_s + 3599) / 3600 )) + rnd=$(( (RANDOM % 4) + 1 )) # 1..4 + total_h=$(( ceil_h + rnd )) + + extra_nbplay_opts+=("-erebootin=${total_h}") + log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)" + elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-reboot" # runs update-reboot.yml (wrapper -> update-rebootin222.yml) + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then + # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin= + n="${BASH_REMATCH[1]}" + base_task="update-indoor" + extra_nbplay_opts+=("-erebootin=${n}") + log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})" + fi + + # --- Default rebootin for reboot-family when not explicitly provided --- + if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot)$ ]]; then + # only set if neither task_options nor extra_nbplay_opts already contain rebootin + if [[ "$task_options" != *"rebootin="* ]] && ! printf '%s\n' "${extra_nbplay_opts[@]}" | grep -q 'rebootin='; then + # restore legacy behavior: immediate reboot if none specified + extra_nbplay_opts+=("-erebootin=0") + log "No rebootin provided; defaulting to rebootin=0 for ${base_task}" + fi + fi + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env. + # This sets RMQ_EXCHANGE=controls only for this nbplay invocation. + if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide source --- +if [[ -n "$EXCHANGE" ]]; then + # Mode B (unchanged): bind a temp queue to exchange+routing key + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" + + # Single-queue consume loop (unchanged branch) + log "Press Ctrl+C to stop." + while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + done + +else + # Mode A: direct queue(s). Support CSV list in QUEUE. + IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE" + for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done + + if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then + err "No queues configured (QUEUE env empty)." + exit 2 + fi + + log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)" + log "Press Ctrl+C to stop." + + # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over. + while :; do + local_got_message=0 + + for Q in "${QUEUE_LIST[@]}"; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" != "[]" && -n "$RESP" ]]; then + local_got_message=1 + log "Dequeued from [$Q]" + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + + # After processing a message from this queue, start the round over (fair-ish polling). + break + fi + done + + if [[ "$local_got_message" -eq 0 ]]; then + sleep "$SLEEP_SECS" + fi + done +fi diff --git a/files/rabbit-client.sh--bbz b/files/rabbit-client.sh--bbz new file mode 100644 index 0000000..a87570a --- /dev/null +++ b/files/rabbit-client.sh--bbz @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Hard override as in your current file +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default). Supports CSV for multiple queues. +# CHANGED: default now includes both normal and persuasive work queues. +QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}" # e.g. "queue_deviceconfig,queue_persuasive" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth) + if [[ "$task" == "afterupgrade_check" ]]; then + local attempt corr_id orig_at target_ver + attempt="$(jq -r '.attempt // empty' <<<"$json")" + corr_id="$(jq -r '.correlation_id // empty' <<<"$json")" + orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")" + target_ver="$(jq -r '.target_version // empty' <<<"$json")" + + [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}" + [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}" + [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}" + + if [[ -n "$target_ver" ]]; then + local esc_tv=${target_ver//\'/\'\"\'\"\'} # replace ' with '\'' safely + task_options+=" -e target_version='${esc_tv}'" + fi + fi + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special cases: parametric reboot hours from task name + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + + # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4 + if [[ "$task" =~ ^(.+)_tonight$ ]]; then + base_task="${BASH_REMATCH[1]}" + + # now, today 01:00, tomorrow 01:00 (local time) + local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h + now_s="$(date +%s)" + today1_s="$(date -d 'today 01:00' +%s)" + tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)" + if (( now_s < today1_s )); then + next1_s="$today1_s" + else + next1_s="$tomorrow1_s" + fi + diff_s=$(( next1_s - now_s )) + # Ceil hours so current minutes are preserved as in your examples + ceil_h=$(( (diff_s + 3599) / 3600 )) + rnd=$(( (RANDOM % 4) + 1 )) # 1..4 + total_h=$(( ceil_h + rnd )) + + extra_nbplay_opts+=("-erebootin=${total_h}") + log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)" + elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-reboot" # runs update-reboot.yml (wrapper -> update-rebootin222.yml) + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then + # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin= + n="${BASH_REMATCH[1]}" + base_task="update-indoor" + extra_nbplay_opts+=("-erebootin=${n}") + log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})" + fi + # --- Default rebootin for reboot-family when not explicitly provided + if [[ "$base_task" =~ ^(update-rebootin222|update-rebootin|update-reboot)$ ]]; then + # only set if neither task_options nor extra_nbplay_opts already contain rebootin + if [[ "$task_options" != *"rebootin="* ]] && ! printf '%s\n' "${extra_nbplay_opts[@]}" | grep -q 'rebootin='; then + # For these playbooks, rebootin is interpreted as hours; 0 == reboot now. + extra_nbplay_opts+=("-e" "rebootin=0") + log "No rebootin provided; defaulting to rebootin=0 for ${base_task}" + fi + fi + + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env. + # This sets RMQ_EXCHANGE=controls only for this nbplay invocation. + if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide source --- +if [[ -n "$EXCHANGE" ]]; then + # Mode B (unchanged): bind a temp queue to exchange+routing key + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" + + # Single-queue consume loop (unchanged branch) + log "Press Ctrl+C to stop." + while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + done + +else + # Mode A: direct queue(s). Support CSV list in QUEUE. + IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE" + for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done + + if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then + err "No queues configured (QUEUE env empty)." + exit 2 + fi + + log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)" + log "Press Ctrl+C to stop." + + # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over. + while :; do + local_got_message=0 + + for Q in "${QUEUE_LIST[@]}"; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" != "[]" && -n "$RESP" ]]; then + local_got_message=1 + log "Dequeued from [$Q]" + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + + # After processing a message from this queue, start the round over (fair-ish polling). + break + fi + done + + if [[ "$local_got_message" -eq 0 ]]; then + sleep "$SLEEP_SECS" + fi + done +fi diff --git a/files/rabbit-client.sh-before_multiqueue b/files/rabbit-client.sh-before_multiqueue new file mode 100644 index 0000000..87a6009 --- /dev/null +++ b/files/rabbit-client.sh-before_multiqueue @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Hard override as in your current file +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default) +QUEUE="${QUEUE:-queue_deviceconfig}" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + # We build an eval that appends parsed words to the array. + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth) + if [[ "$task" == "afterupgrade_check" ]]; then + local attempt corr_id orig_at target_ver + attempt="$(jq -r '.attempt // empty' <<<"$json")" + corr_id="$(jq -r '.correlation_id // empty' <<<"$json")" + orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")" + target_ver="$(jq -r '.target_version // empty' <<<"$json")" + + # Build -e flags into task_options (respect existing task_options) + [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}" + [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}" + [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}" + + # target_version may contain spaces (e.g., "2.2.2 rev 9778") + # append_opts uses eval, so wrap in single quotes and escape embedded single quotes. + if [[ -n "$target_ver" ]]; then + local esc_tv=${target_ver//\'/\'\"\'\"\'} # replace ' with '\'' safely + task_options+=" -e target_version='${esc_tv}'" + fi + fi + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special case: task "update-rebootin_N" (N=0..99) + # If matched, run playbook "update-rebootin.yml" and append "-erebootin=N" at the end of nbplay args. + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + fi + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + # Build command array and append options (respect quoted args) + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + # Append any special-case options at the very end (e.g., -erebootin=N) + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + if ! "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide queue source --- +if [[ -n "$EXCHANGE" ]]; then + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" +else + log "Consuming directly from queue: $QUEUE (vhost: $VHOST)" +fi + +log "Press Ctrl+C to stop." + +# --- Consume loop --- +while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + # Empty array => no messages + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + # Process one message (array of length 1) + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + + # Decode if payload is a quoted JSON string + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + # Fallback: cannot parse JSON without jq + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi +done diff --git a/files/rabbit-client.sh-beforedelayed b/files/rabbit-client.sh-beforedelayed new file mode 100644 index 0000000..8783110 --- /dev/null +++ b/files/rabbit-client.sh-beforedelayed @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default) +QUEUE="${QUEUE:-queue_deviceconfig}" + + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + # We build an eval that appends parsed words to the array. + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special case: task "update-rebootin_N" (N=0..99) + # If matched, run playbook "update-rebootin.yml" and append "-erebootin=N" at the end of nbplay args. + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + fi + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + # Build command array and append options (respect quoted args) + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + # Append any special-case options at the very end (e.g., -erebootin=N) + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + if ! "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide queue source --- +if [[ -n "$EXCHANGE" ]]; then + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" +else + log "Consuming directly from queue: $QUEUE (vhost: $VHOST)" +fi + +log "Press Ctrl+C to stop." + +# --- Consume loop --- +while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + # Empty array => no messages + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + # Process one message (array of length 1) + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + + # Decode if payload is a quoted JSON string + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + # Fallback: cannot parse JSON without jq + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi +done diff --git a/files/rabbit-client.sh-beforequeuefixing b/files/rabbit-client.sh-beforequeuefixing new file mode 100644 index 0000000..f39fa2d --- /dev/null +++ b/files/rabbit-client.sh-beforequeuefixing @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Hard override as in your current file +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default). Supports CSV for multiple queues. +QUEUE="${QUEUE:-queue_deviceconfig}" # e.g. "queue_deviceconfig,queue_persuasive" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth) + if [[ "$task" == "afterupgrade_check" ]]; then + local attempt corr_id orig_at target_ver + attempt="$(jq -r '.attempt // empty' <<<"$json")" + corr_id="$(jq -r '.correlation_id // empty' <<<"$json")" + orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")" + target_ver="$(jq -r '.target_version // empty' <<<"$json")" + + [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}" + [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}" + [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}" + + if [[ -n "$target_ver" ]]; then + local esc_tv=${target_ver//\'/\'\"\'\"\'} # replace ' with '\'' safely + task_options+=" -e target_version='${esc_tv}'" + fi + fi + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special case: task "update-rebootin_N" (N=0..99) + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + fi + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + if ! "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide source --- +if [[ -n "$EXCHANGE" ]]; then + # Mode B (unchanged): bind a temp queue to exchange+routing key + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" + + # Single-queue consume loop (unchanged branch) + log "Press Ctrl+C to stop." + while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + done + +else + # Mode A: direct queue(s). Support CSV list in QUEUE. + IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE" + for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done + + if ((${#QUEUE_LIST[@]} == 0)); then + err "No queues configured (QUEUE env empty)." + exit 2 + fi + + log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)" + log "Press Ctrl+C to stop." + + # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over. + while :; do + local_got_message=0 + + for Q in "${QUEUE_LIST[@]}"; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" != "[]" && -n "$RESP" ]]; then + local_got_message=1 + log "Dequeued from [$Q]" + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + + # After processing a message from this queue, start the round over (fair-ish polling). + break + fi + done + + if [[ "$local_got_message" -eq 0 ]]; then + sleep "$SLEEP_SECS" + fi + done +fi diff --git a/files/rabbit-client.sh-fixing-rebootin b/files/rabbit-client.sh-fixing-rebootin new file mode 100644 index 0000000..f7ea664 --- /dev/null +++ b/files/rabbit-client.sh-fixing-rebootin @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Hard override as in your current file +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default). Supports CSV for multiple queues. +# CHANGED: default now includes both normal and persuasive work queues. +QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}" # e.g. "queue_deviceconfig,queue_persuasive" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth) + if [[ "$task" == "afterupgrade_check" ]]; then + local attempt corr_id orig_at target_ver + attempt="$(jq -r '.attempt // empty' <<<"$json")" + corr_id="$(jq -r '.correlation_id // empty' <<<"$json")" + orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")" + target_ver="$(jq -r '.target_version // empty' <<<"$json")" + + [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}" + [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}" + [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}" + + if [[ -n "$target_ver" ]]; then + local esc_tv=${target_ver//\'/\'\"\'\"\'} # replace ' with '\'' safely + task_options+=" -e target_version='${esc_tv}'" + fi + fi + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special cases: parametric reboot hours from task name + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + if [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-reboot" # runs update-reboot.yml (wrapper -> update-rebootin222.yml) + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-indoor" + # Convert hours -> minutes for nbplay's --rebootin + minutes=$((10#$n * 60)) + extra_nbplay_opts+=("--rebootin" "${minutes}") + log "Parsed update-indoor suffix: ${n}h -> ${minutes}m (rebootin)" + fi + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env. + # This sets RMQ_EXCHANGE=controls only for this nbplay invocation. + if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide source --- +if [[ -n "$EXCHANGE" ]]; then + # Mode B (unchanged): bind a temp queue to exchange+routing key + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" + + # Single-queue consume loop (unchanged branch) + log "Press Ctrl+C to stop." + while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + done + +else + # Mode A: direct queue(s). Support CSV list in QUEUE. + IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE" + for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done + + if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then + err "No queues configured (QUEUE env empty)." + exit 2 + fi + + log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)" + log "Press Ctrl+C to stop." + + # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over. + while :; do + local_got_message=0 + + for Q in "${QUEUE_LIST[@]}"; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" != "[]" && -n "$RESP" ]]; then + local_got_message=1 + log "Dequeued from [$Q]" + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + + # After processing a message from this queue, start the round over (fair-ish polling). + break + fi + done + + if [[ "$local_got_message" -eq 0 ]]; then + sleep "$SLEEP_SECS" + fi + done +fi diff --git a/files/rabbit-client.sh-fixing-rebootin2 b/files/rabbit-client.sh-fixing-rebootin2 new file mode 100644 index 0000000..adfc411 --- /dev/null +++ b/files/rabbit-client.sh-fixing-rebootin2 @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Hard override as in your current file +RMQ_HOST="10.210.12.2" + +# Mode A: read from a queue (default). Supports CSV for multiple queues. +# CHANGED: default now includes both normal and persuasive work queues. +QUEUE="${QUEUE:-queue_deviceconfig,queue_persuasive}" # e.g. "queue_deviceconfig,queue_persuasive" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # empty => Mode B OFF +ROUTING_KEY="${ROUTING_KEY:-}" # irrelevant when EXCHANGE is empty + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Paths +APP_ROOT="/opt/containers/ansible-worker/app" +NBPLAY="${APP_ROOT}/bin/nbplay" + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } + +log() { printf '[consumer] %s\n' "$*"; } +warn() { printf '[consumer][WARN] %s\n' "$*" >&2; } +err() { printf '[consumer][ERROR] %s\n' "$*" >&2; } + +# Safely append string options into an array using eval (so quotes are honored). +append_opts() { + local opts_str="$1" + # shellcheck disable=SC2206 + local -n _arr_ref=$2 + if [[ -n "$opts_str" ]]; then + eval '_arr_ref+=('"$opts_str"')' + fi +} + +# Given a JSON object payload, extract fields and dispatch nbplay +dispatch_task() { + local json="$1" + local device task playbook task_options + device="$(jq -er '.inscope_device // empty' <<<"$json")" || device="" + task="$(jq -er '.task_name // empty' <<<"$json")" || task="" + task_options="$(jq -r '.task_options // empty' <<<"$json")" || task_options="" + + # Pass after-upgrade metadata via -e by augmenting task_options (single source of truth) + if [[ "$task" == "afterupgrade_check" ]]; then + local attempt corr_id orig_at target_ver + attempt="$(jq -r '.attempt // empty' <<<"$json")" + corr_id="$(jq -r '.correlation_id // empty' <<<"$json")" + orig_at="$(jq -r '.original_emitted_at // empty' <<<"$json")" + target_ver="$(jq -r '.target_version // empty' <<<"$json")" + + [[ -n "$attempt" ]] && task_options+=" -e attempt=${attempt}" + [[ -n "$corr_id" ]] && task_options+=" -e correlation_id=${corr_id}" + [[ -n "$orig_at" ]] && task_options+=" -e original_emitted_at=${orig_at}" + + if [[ -n "$target_ver" ]]; then + local esc_tv=${target_ver//\'/\'\"\'\"\'} # replace ' with '\'' safely + task_options+=" -e target_version='${esc_tv}'" + fi + fi + + if [[ -z "$device" || -z "$task" ]]; then + warn "payload missing required keys (inscope_device/task_name). Skipping." + return 0 + fi + + # --- Special cases: parametric reboot hours from task name + local base_task="$task" + local -a extra_nbplay_opts=() + local n="" + + # NEW: generic *_tonight → compute hours until next 01:00 (ceil) + random 1..4 + if [[ "$task" =~ ^(.+)_tonight$ ]]; then + base_task="${BASH_REMATCH[1]}" + + # now, today 01:00, tomorrow 01:00 (local time) + local now_s today1_s tomorrow1_s next1_s diff_s ceil_h rnd extra_h total_h + now_s="$(date +%s)" + today1_s="$(date -d 'today 01:00' +%s)" + tomorrow1_s="$(date -d 'tomorrow 01:00' +%s)" + if (( now_s < today1_s )); then + next1_s="$today1_s" + else + next1_s="$tomorrow1_s" + fi + diff_s=$(( next1_s - now_s )) + # Ceil hours so current minutes are preserved as in your examples + ceil_h=$(( (diff_s + 3599) / 3600 )) + rnd=$(( (RANDOM % 4) + 1 )) # 1..4 + total_h=$(( ceil_h + rnd )) + + extra_nbplay_opts+=("-erebootin=${total_h}") + log "Resolved '${task}' → base='${base_task}', rebootin=${total_h}h (ceil_to_1am=${ceil_h}h + rand=${rnd}h)" + elif [[ "$task" =~ ^update-rebootin_([0-9]{1,2})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-rebootin" + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-reboot_([0-9]{1,4})$ ]]; then + n="${BASH_REMATCH[1]}" + base_task="update-reboot" # runs update-reboot.yml (wrapper -> update-rebootin222.yml) + extra_nbplay_opts+=("-erebootin=${n}") + elif [[ "$task" =~ ^update-indoor_([0-9]{1,4})$ ]]; then + # CHANGED: keep same convention as others — pass HOURS directly via -e rebootin= + n="${BASH_REMATCH[1]}" + base_task="update-indoor" + extra_nbplay_opts+=("-erebootin=${n}") + log "Parsed update-indoor suffix: ${n}h (passed as -e rebootin=${n})" + fi + + playbook="${APP_ROOT}/${base_task}.yml" + if [[ ! -f "$playbook" ]]; then + warn "playbook not found: $playbook (device='$device', task='$task'). Skipping." + return 0 + fi + + log "→ Running $(basename "$playbook") on '${device}' with options: ${task_options:-}" + + local -a cmd=( "$NBPLAY" "$playbook" "$device" ) + append_opts "$task_options" cmd + + if ((${#extra_nbplay_opts[@]})); then + cmd+=("${extra_nbplay_opts[@]}") + fi + + # Ensure playbooks publish control/journal/tag to the correct exchange without requiring container env. + # This sets RMQ_EXCHANGE=controls only for this nbplay invocation. + if ! RMQ_EXCHANGE=controls "${cmd[@]}"; then + err "playbook failed (task='${task}', device='${device}')" + # keep consuming; message already acked by HTTP get endpoint + fi +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + log "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide source --- +if [[ -n "$EXCHANGE" ]]; then + # Mode B (unchanged): bind a temp queue to exchange+routing key + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + log "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, "durable": false, "arguments": {}, "exclusive": true + }' >/dev/null + + log "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + log "Consuming from bound temp queue: $QUEUE" + + # Single-queue consume loop (unchanged branch) + log "Press Ctrl+C to stop." + while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" == "[]" || -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + done + +else + # Mode A: direct queue(s). Support CSV list in QUEUE. + IFS=',' read -r -a QUEUE_LIST <<< "$QUEUE" + for i in "${!QUEUE_LIST[@]}"; do QUEUE_LIST[$i]="${QUEUE_LIST[$i]//[[:space:]]/}"; done + + if [[ ${#QUEUE_LIST[@]} -eq 0 ]]; then + err "No queues configured (QUEUE env empty)." + exit 2 + fi + + log "Consuming from queue(s): ${QUEUE_LIST[*]} (vhost: $VHOST)" + log "Press Ctrl+C to stop." + + # Multi-queue round-robin: try each queue once per loop; if any yields a message, process it and start over. + while :; do + local_got_message=0 + + for Q in "${QUEUE_LIST[@]}"; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$Q/get" '{ + "count": 1, "ackmode": "ack_requeue_false", "encoding": "auto", "truncate": 1000000 + }')" + + if [[ "$RESP" != "[]" && -n "$RESP" ]]; then + local_got_message=1 + log "Dequeued from [$Q]" + if has_jq; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + payload=$(printf '%s' "$item" | jq -r '.payload') + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + dispatch_task "$decoded" + else + warn "payload is a string but not JSON after unquote; skipping." + fi + else + dispatch_task "$payload" + fi + else + warn "payload is not JSON; skipping." + fi + done + else + warn "jq not found; cannot parse JSON payloads. Exiting." + exit 3 + fi + + # After processing a message from this queue, start the round over (fair-ish polling). + break + fi + done + + if [[ "$local_got_message" -eq 0 ]]; then + sleep "$SLEEP_SECS" + fi + done +fi diff --git a/files/rabbit-consumer.sh b/files/rabbit-consumer.sh new file mode 100644 index 0000000..1c070fd --- /dev/null +++ b/files/rabbit-consumer.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Config (override via env or flags) --- +RMQ_USER="${RMQ_USER:-admin}" +RMQ_PASS="${RMQ_PASS:-change_me}" +RMQ_HOST="${RMQ_HOST:-localhost}" +RMQ_PORT="${RMQ_PORT:-15672}" +VHOST="${VHOST:-app}" + +# Mode A: read from a queue (default) +QUEUE="${QUEUE:-queue1}" + +# Mode B: bind temp queue to an exchange + routing key and consume from it +EXCHANGE="${EXCHANGE:-}" # e.g. "amq.topic" or "my-exchange". Leave empty to skip binding mode. +ROUTING_KEY="${ROUTING_KEY:-#}" # pattern for topic/direct, default is catch-all "#" + +# Polling interval when no messages +SLEEP_SECS="${SLEEP_SECS:-1}" + +# Pretty-print with jq if available +USE_JQ="${USE_JQ:-auto}" # auto|yes|no +USE_COLOR="${USE_COLOR:-yes}" # yes|no -> 'yes' forces color with jq -C + +# --- Helpers --- +api() { + local method="$1"; shift + local path="$1"; shift + local data="${1:-}" + if [[ -n "$data" ]]; then + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" -d "$data" + else + curl -sS -u "$RMQ_USER:$RMQ_PASS" -H "content-type:application/json" -X "$method" "http://$RMQ_HOST:$RMQ_PORT$path" + fi +} + +has_jq() { command -v jq >/dev/null 2>&1; } + +# pretty(): colorized JSON when jq is present. We **force** color with -C so it survives pipes/subshells. +pretty() { + if [[ "$USE_JQ" == "yes" ]] || { [[ "$USE_JQ" == "auto" ]] && has_jq; }; then + if [[ "$USE_COLOR" == "yes" ]]; then + jq -C . + else + jq . + fi + else + cat + fi +} + +urlenc() { + # vhost is simple ("app"), so we skip full encoding, but keep function for completeness + printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g' +} + +cleanup_queue="" +cleanup() { + if [[ -n "$cleanup_queue" ]]; then + echo "Cleaning up temp queue: $cleanup_queue" + api DELETE "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" >/dev/null || true + fi +} +trap cleanup EXIT + +# --- Setup: decide queue source --- +if [[ -n "$EXCHANGE" ]]; then + # Create a temp, exclusive, auto-delete queue and bind to exchange + cleanup_queue="tmp.$(hostname -s).$$.$(date +%s)" + echo "Declaring temp queue: $cleanup_queue (exclusive, auto-delete)" + api PUT "/api/queues/$(urlenc "$VHOST")/$cleanup_queue" '{ + "auto_delete": true, + "durable": false, + "arguments": {}, + "exclusive": true + }' >/dev/null + + echo "Binding temp queue to exchange '$EXCHANGE' with routing key '$ROUTING_KEY'" + api POST "/api/bindings/$(urlenc "$VHOST")/e/$EXCHANGE/q/$cleanup_queue" "{ + \"routing_key\": \"$ROUTING_KEY\", + \"arguments\": {} + }" >/dev/null + + QUEUE="$cleanup_queue" + echo "Consuming from bound temp queue: $QUEUE" +else + echo "Consuming directly from queue: $QUEUE (vhost: $VHOST)" +fi + +echo "Press Ctrl+C to stop." + +# --- Consume loop --- +while :; do + RESP="$(api POST "/api/queues/$(urlenc "$VHOST")/$QUEUE/get" '{ + "count": 1, + "ackmode": "ack_requeue_false", + "encoding": "auto", + "truncate": 1000000 + }')" + + # Empty array => no messages + if [[ "$RESP" == "[]" ]] || [[ -z "$RESP" ]]; then + sleep "$SLEEP_SECS" + continue + fi + + # Print one-by-one (the API returns an array) + if has_jq && { [[ "$USE_JQ" == "yes" ]] || [[ "$USE_JQ" == "auto" ]]; }; then + echo "$RESP" | jq -c '.[]' | while read -r item; do + # Extract fields (raw to avoid extra quotes) + payload=$(printf '%s' "$item" | jq -r '.payload') + rk=$(printf '%s' "$item" | jq -r '.routing_key') + ex=$(printf '%s' "$item" | jq -r '.exchange') + + echo "-----" + echo "exchange: ${ex:-\"\"}" + echo "routing_key: $rk" + echo "payload:" + + # Smart payload handling: + # 1) If payload parses as JSON: + # - If it's a JSON string, decode once; if decoded text is JSON, pretty-print it; else print plain text. + # - If it's object/array/etc, pretty-print directly. + # 2) If payload isn't JSON at all, print as-is. + if jq -e . >/dev/null 2>&1 <<<"$payload"; then + ptype="$(printf '%s' "$payload" | jq -r 'type')" + if [[ "$ptype" == "string" ]]; then + decoded="$(printf '%s' "$payload" | jq -r .)" + if jq -e . >/dev/null 2>&1 <<<"$decoded"; then + printf '%s' "$decoded" | pretty + else + printf '%s\n' "$decoded" + fi + else + printf '%s' "$payload" | pretty + fi + else + printf '%s\n' "$payload" + fi + done + else + # Minimal parsing without jq + echo "$RESP" + fi +done diff --git a/files/ssh.yml b/files/ssh.yml new file mode 100644 index 0000000..ed43a3c --- /dev/null +++ b/files/ssh.yml @@ -0,0 +1,3 @@ +ansible_user: root +ansible_ssh_pass: wavewave +