1001

files/ansible-playbooks/update-indoor.yml

@@ -50,11 +50,6 @@
     control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"
     afterupgrade_routing_key: "{{ lookup('env','AFTERUP_ROUTING_KEY') | default('deviceconfig', true) }}"
 
-    # --- TEMP: stall-test knobs (remove later) ---
-    aic_stall_test: true        # enable/disable the simulated stall on DEV2
-    aic_stall_sleep: 1000       # seconds to sleep on DEV2 (simulate hang)
-    aic_stall_async: 10         # async timeout (seconds) for the sleep job on DEV2
-    aic_stall_poll: 2           # poll interval (seconds) while watching the job
     
   pre_tasks:
     # ------------------------------- Hostname sanity DEV1 -------------------------------
@@ -461,63 +456,6 @@
         _blocked: true
         _journal: "{{ _journal + [ 'Preparation markers already present on DEV2 (count=' ~ (dev2_prep_count.stdout | trim) ~ '). Skipping staging/write' ] }}"
 
-    # test test test
-
-    # === TEMP: STALL TEST ON DEV2 (BEGIN) ========================================
-    # Enable with:  -e aic_stall_test=true
-    # Optional knobs: -e aic_stall_sleep=1000 -e aic_stall_async=10 -e aic_stall_poll=2
-
-    - name: TEMP | Set stall test toggles from env/defaults
-      delegate_to: localhost
-      ansible.builtin.set_fact:
-        aic_stall_test: "{{ (aic_stall_test | default(lookup('env','AIC_STALL_TEST') | default('false', true))) | bool }}"
-        aic_stall_sleep: "{{ (aic_stall_sleep | default(lookup('env','AIC_STALL_SLEEP') | default(1000, true))) | int }}"
-        aic_stall_async: "{{ (aic_stall_async | default(lookup('env','AIC_STALL_ASYNC') | default(10, true))) | int }}"
-        aic_stall_poll:  "{{ (aic_stall_poll  | default(lookup('env','AIC_STALL_POLL')  | default(2, true))) | int }}"
-
-    - name: TEMP | Announce stall test settings
-      when: aic_stall_test
-      delegate_to: localhost
-      ansible.builtin.debug:
-        msg:
-          - "STALL TEST ENABLED"
-          - "sleep={{ aic_stall_sleep }}s  async_cap={{ aic_stall_async }}s  poll={{ aic_stall_poll }}s"
-
-    - name: TEMP | Simulate a stuck operation on DEV2 via tunnel (async-capped)
-      when: aic_stall_test
-      delegate_to: localhost
-      ansible.builtin.shell: |
-        set -e
-        PORT="{{ _local_port }}"
-        sshpass -f "{{ dev2_passfile_used }}" ssh \
-          -o AddressFamily=inet \
-          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-          -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-          -o ConnectTimeout=10 \
-          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-          "sh -c 'date; echo STALL-TEST-START; sleep {{ aic_stall_sleep }}; echo STALL-TEST-END; date'"
-      args:
-        executable: /bin/bash
-      async: "{{ aic_stall_async }}"
-      poll: "{{ aic_stall_poll }}"
-      register: stall_test_result
-      changed_when: false
-      failed_when: false   # capture outcome; don't fail the whole play
-
-    - name: TEMP | Show stall test result (expect timeout-ish when sleep > async)
-      when: aic_stall_test
-      delegate_to: localhost
-      ansible.builtin.debug:
-        var: stall_test_result
-
-    - name: TEMP | Exit early after stall test (no device actions beyond this point)
-      when: aic_stall_test
-      ansible.builtin.meta: end_play
-    # === TEMP: STALL TEST ON DEV2 (END) ==========================================
-
-
-    # test test test
-
     # ---------------------------- DEV2 version firmux primary check ----------------------------
     - name: Read DEV2 /usr/lib/release/firmux (if present)
       when: dev2_passfile_used != "NONE"
@@ -856,25 +794,102 @@
     - name: Upgrade write and bank flip on DEV2 (guarded by soft-block)
       when: not (_blocked | default(false))
       block:
-        - name: Write image to inactive bank on DEV2 (update -w)
-          delegate_to: localhost
-          ansible.builtin.shell: |
-            set -e
-            PORT="{{ _local_port }}"
-            sshpass -f "{{ dev2_passfile_used }}" ssh \
-              -o AddressFamily=inet \
-              -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
-              -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
-              -o ConnectTimeout=0 \
-              -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
-              "update -w '{{ dev2_image_path }}'"
-          args:
-            executable: /bin/bash
-          register: dev2_up_write
-          changed_when: true
-          failed_when: dev2_up_write.stdout is not search('update is complete')
 
+        # --- supervised async write with stall handling ---
+        - name: Write image to inactive bank on DEV2 (update -w, supervised async)
+          delegate_to: localhost
+          vars:
+            _write_async_cap: 600     # 10 minutes max runtime (tune if needed)
+            _write_poll: 20           # poll every 20 seconds
+          block:
+            - name: Launch update -w on DEV2 (async)
+              ansible.builtin.shell: |
+                set -e
+                PORT="{{ _local_port }}"
+                sshpass -f "{{ dev2_passfile_used }}" ssh \
+                  -o AddressFamily=inet \
+                  -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+                  -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                  -o ConnectTimeout=0 \
+                  -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                  "update -w '{{ dev2_image_path }}'"
+              args:
+                executable: /bin/bash
+              async: "{{ _write_async_cap }}"
+              poll: "{{ _write_poll }}"
+              register: dev2_up_write
+              changed_when: true
+              failed_when: false
+
+            - name: Evaluate async result
+              ansible.builtin.set_fact:
+                _updatew_stalled: >-
+                  {{ (dev2_up_write.msg is defined) and
+                     ('did not complete' in dev2_up_write.msg | lower) }}
+                _updatew_success: >-
+                  {{ (dev2_up_write.stdout | default('')) is search('update is complete') }}
+
+            - name: Journal stall if update -w timed out
+              when: _updatew_stalled | bool
+              ansible.builtin.set_fact:
+                journal_updatew_stalled:
+                  inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
+                  task_name: "journal_add"
+                  task_result: "update -w has stalled. Rebooting device; please schedule another upgrade."
+
+            - name: Publish stall journal to control queue
+              when: journal_updatew_stalled is defined
+              delegate_to: localhost
+              ansible.builtin.uri:
+                url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
+                method: POST
+                user: "{{ rmq_user }}"
+                password: "{{ rmq_pass }}"
+                force_basic_auth: true
+                status_code: 200
+                headers:
+                  content-type: "application/json"
+                body_format: json
+                body:
+                  properties:
+                    content_type: "application/json"
+                  routing_key: "{{ control_queue }}"
+                  payload: "{{ journal_updatew_stalled | to_json }}"
+                  payload_encoding: "string"
+              register: rmq_stall_journal_resp
+              changed_when: (rmq_stall_journal_resp.json is defined) and (rmq_stall_journal_resp.json.routed | default(false) | bool)
+              failed_when: false
+
+            - name: Schedule immediate reboot (20s delay) after stall
+              when: _updatew_stalled | bool
+              delegate_to: localhost
+              ansible.builtin.shell: |
+                set -e
+                PORT="{{ _local_port }}"
+                sshpass -f "{{ dev2_passfile_used }}" ssh \
+                  -o AddressFamily=inet \
+                  -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
+                  -o PreferredAuthentications=password -o NumberOfPasswordPrompts=1 \
+                  -o ConnectTimeout=10 \
+                  -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
+                  "nohup /sbin/reboot -d 20 >/dev/null 2>&1 &"
+              args:
+                executable: /bin/bash
+              changed_when: true
+              ignore_errors: true
+
+            - name: Stop play if update -w stalled
+              when: _updatew_stalled | bool
+              ansible.builtin.meta: end_play
+
+            - name: Fail play if update -w completed but invalid output
+              when: (not _updatew_stalled) and (not _updatew_success)
+              ansible.builtin.fail:
+                msg: "update -w finished but did not report 'update is complete'"
+
+        # --- continue normal path if not stalled ---
         - name: Read current active partition on DEV2 (before flip)
+          when: not _updatew_stalled | bool
           delegate_to: localhost
           ansible.builtin.shell: |
             set -e
@@ -893,12 +908,12 @@
           failed_when: (dev2_active_before.stdout | trim) not in ['1','2']
 
         - name: Determine new active value for DEV2 (flip 1↔2)
-          when: dev2_active_before is defined and (dev2_active_before.stdout is defined) and ((dev2_active_before.stdout | trim) in ['1','2'])
+          when: not _updatew_stalled | bool and dev2_active_before.stdout is defined
           ansible.builtin.set_fact:
             dev2_new_active: "{{ '1' if (dev2_active_before.stdout | trim) == '2' else '2' }}"
 
         - name: Set fw_setenv active={{ dev2_new_active }} on DEV2
-          when: dev2_new_active is defined
+          when: not _updatew_stalled | bool and dev2_new_active is defined
           delegate_to: localhost
           ansible.builtin.shell: |
             set -e
@@ -916,7 +931,7 @@
           changed_when: true
 
         - name: Verify active partition flipped on DEV2 (read back)
-          when: dev2_new_active is defined
+          when: not _updatew_stalled | bool and dev2_new_active is defined
           delegate_to: localhost
           ansible.builtin.shell: |
             set -e
@@ -935,6 +950,7 @@
           failed_when: (dev2_active_after.stdout | trim) != (dev2_new_active | string)
 
         - name: Create specific prep marker on DEV2 for this image
+          when: not _updatew_stalled | bool
           delegate_to: localhost
           ansible.builtin.shell: |
             set -e
@@ -952,6 +968,8 @@
           changed_when: true
           ignore_errors: true
    
+    
+
     # ---------------------------- Reboot scheduling (normalized) ----------------------------
     - name: Schedule DEV2 reboot after computed delay (seconds)
       when: _reboot_requested and (_reboot_minutes | int) >= 0 and dev2_passfile_used != "NONE" and not (_blocked | default(false))