---
- name: After-upgrade verification (banner check + reporting)
  hosts: all
  gather_facts: no

  # RabbitMQ + defaults (match the big script)
  vars:
    rmq_host: "10.210.12.2"
    rmq_port: 15672
    rmq_user: "admin"
    rmq_pass: "change_me"
    rmq_vhost: "app"
    rmq_exchange: "controls"
    control_queue: "queue_controls"

# --- Manual run defaults (so we can execute without -e) --- 
# These are safe to leave here; anything passed via -e will still override them.
    attempt: 1 
    max_attempts: 3 
    current_delay_sec: 300 
    correlation_id: "6f680073dc7c" 
    original_emitted_at: "2025-10-30T18:46:52Z" 
    target_version: "2.2.3 rev 9800" 
    # Intentionally keep this empty to exercise the target_version_effective logic. 
    target_version_full: "" 
    schema_version: 1


    # Probing/SSH defaults
    tcp_port: 22
    nc_timeout: 5
    ssh_user: "{{ ansible_user | default('root') }}"
    ssh_pass: "{{ ansible_ssh_pass | default('wavewave') }}"
    ssh_timeout: 10

    # Do NOT self-reference max_attempts. We’ll normalize below.
    max_attempts_default: 3

    # --- Hardcoded cloud API bearer (per request) ---
    cloud_api_bearer: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InBhdmVsLmxAOGRldmljZXMuY29tIiwic3ViIjoyMiwiaWF0IjoxNzg2NjAxMDAwLCJleHAiOjE3ODkxOTMwMDB9.NHGZSl9GDTRF-U12vIZnfxuOyTl_Q0JM8x_jkVBCgP8"

  tasks:
    # ---- Normalize metadata safely (no self-referential defaults) ----
    - name: Normalize metadata (no clever transforms)
      ansible.builtin.set_fact:
        attempt: "{{ (attempt | default(1)) | int }}"
        effective_max_attempts: "{{ (max_attempts | default(max_attempts_default)) | int }}"
        correlation_id: "{{ correlation_id | default('') }}"
        original_emitted_at: "{{ original_emitted_at | default('') }}"
        target_version: "{{ target_version | default('') }}"
        # preserve the original string verbatim for all subsequent retries
        target_version_full: "{{ target_version | default('') }}"

    - name: Derive effective target version (avoid extra-var masking)
      ansible.builtin.set_fact:
        target_version_effective: >-
          {{ (target_version_full | default('') | trim)
              if (target_version_full | default('') | trim)
              else (target_version | default('') | trim) }}

    - name: Show received metadata
      ansible.builtin.debug:
        msg:
          - "attempt={{ attempt }}"
          - "max_attempts={{ effective_max_attempts }}"
          - "correlation_id={{ correlation_id }}"
          - "original_emitted_at={{ original_emitted_at }}"
          - "target_version(full)={{ target_version_full }}"

    # ---- Fast TCP reachability probe (controller-side) ----
    - name: Check if TCP/{{ tcp_port }} is reachable with nc
      delegate_to: localhost
      ansible.builtin.shell: |
        nc -z -w{{ nc_timeout }} {{ ansible_host | default(inventory_hostname) }} {{ tcp_port }}
      register: nc_probe
      changed_when: false
      ignore_errors: true

    - name: Build failure journal (no TCP connectivity) + mark retry
      when: nc_probe.rc != 0
      delegate_to: localhost
      ansible.builtin.set_fact:
        fail_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            afterupgrade_check (attempt {{ attempt }}/{{ effective_max_attempts }}): TCP {{ tcp_port }} unreachable (nc failed).
            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
          _needs_retry: true

    - name: Publish failure journal (no TCP connectivity)
      when: nc_probe.rc != 0
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ fail_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_j_tcp_fail
      changed_when: (rmq_j_tcp_fail.json is defined) and (rmq_j_tcp_fail.json.routed | default(false) | bool)

    # If TCP failed, we do NOT try SSH. We go straight to scheduling (or final “gave up”).
    - name: Stop host after TCP failure (we’ll schedule or close out below)
      when: nc_probe.rc != 0
      ansible.builtin.meta: noop

    # ---- SSH banner probe (controller-side) using the ORIGINAL extraction ----
    - name: Probe banner via SSH from controller (classic extraction)
      when: nc_probe.rc == 0
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        USER="{{ ssh_user }}"
        HOST="{{ ansible_host | default(inventory_hostname) }}"
        sshpass -p '{{ ssh_pass }}' \
        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
          "${USER}@${HOST}" \
          "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; cat /etc/banner | grep -i rev | head -n1"
      register: banner_probe
      changed_when: false
      ignore_errors: true

    - name: Show the current version (banner line)
      when: nc_probe.rc == 0 and banner_probe.rc == 0
      ansible.builtin.debug:
        msg: "{{ banner_probe.stdout | trim }}"

    - name: Evaluate version match (full-string contains check)
      when: nc_probe.rc == 0 and banner_probe.rc == 0
      ansible.builtin.set_fact:
        version_match: "{{ (target_version_effective | length > 0) and (target_version_effective in (banner_probe.stdout | default(''))) }}"

    # ---- Read eth0 MAC (only after confirmed version match) ----
    - name: Read eth0 MAC address via SSH
      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        USER="{{ ssh_user }}"
        HOST="{{ ansible_host | default(inventory_hostname) }}"
        sshpass -p '{{ ssh_pass }}' \
        ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no -o ConnectTimeout={{ ssh_timeout }} \
          "${USER}@${HOST}" \
          "cat /sys/class/net/eth0/address | tr -d '\n'"
      register: mac_probe
      changed_when: false
      ignore_errors: true

    - name: Set eth0_macaddress fact
      when: mac_probe is defined and mac_probe.rc == 0
      ansible.builtin.set_fact:
        eth0_macaddress: "{{ (mac_probe.stdout | default('') | trim) }}"

    - name: Debug eth0_macaddress
      when: eth0_macaddress is defined
      ansible.builtin.debug:
        msg: "eth0_macaddress={{ eth0_macaddress }}"

    # ---- Cloud bandwidth-control PATCH (only after match & MAC present) ----
    - name: Build URL-encoded MAC for cloud API
      when: eth0_macaddress is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
      ansible.builtin.set_fact:
        enc_mac: "{{ eth0_macaddress | regex_replace(':', '%3A') }}"

    - name: PATCH bandwidth-control in cloud (egress 30 / ingress 10)
      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
      delegate_to: localhost
      ansible.builtin.shell: |
        set -e
        curl -sS -L --request PATCH --post301 --post302 \
          "https://cloud.ikeja.co.za/v1/external/devices/{{ enc_mac }}/bandwidth-control" \
          --header "Authorization: Bearer {{ cloud_api_bearer }}" \
          --header "Content-Type: application/json" \
          --header "Accept: application/json" \
          --fail-with-body \
          --data '{"egress":{"isEnabled":true,"speedMbps":30},"ingress":{"isEnabled":true,"speedMbps":10}}'
      register: cloud_patch
      changed_when: false
      ignore_errors: true

    - name: Flag cloud change result
      when: enc_mac is defined and (version_match | bool) and nc_probe.rc == 0 and banner_probe.rc == 0
      ansible.builtin.set_fact:
        cloud_change_ok: "{{ (cloud_patch is defined and (cloud_patch.rc | default(1)) == 0) }}"

    - name: Debug cloud change result
      when: cloud_change_ok is defined
      ansible.builtin.debug:
        msg: "cloud_change={{ 'Ok' if cloud_change_ok else 'NOT ok' }}"

    # ---- Journaling paths ----
    # Success: banner matches expected full target_version
    - name: Build success journal payload
      when: nc_probe.rc == 0 and banner_probe.rc == 0 and (version_match | bool)
      delegate_to: localhost
      ansible.builtin.set_fact:
        journal_success_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            afterupgrade_check SUCCESS (attempt {{ attempt }}/{{ effective_max_attempts }}):
            Banner='{{ (banner_probe.stdout | default('') | trim) }}' Target='{{ target_version_effective }}'
            Correlation={{ correlation_id }} Original={{ original_emitted_at }}
            {{ 'cloud change Ok' if (cloud_change_ok | default(false)) else 'cloud change NOT ok' }}

    - name: Publish success journal to control queue
      when: journal_success_payload is defined
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_success_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_j_success
      changed_when: (rmq_j_success.json is defined) and (rmq_j_success.json.routed | default(false) | bool)

    # NEW: send a control tag to clean up device state on success
    - name: Build cleanup control payload (update_cleanup_success)
      when: journal_success_payload is defined
      delegate_to: localhost
      ansible.builtin.set_fact:
        control_cleanup_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "update_cleanup_success"

    - name: Publish cleanup control message to control queue
      when: control_cleanup_payload is defined
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ control_cleanup_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_cleanup_success
      changed_when: (rmq_cleanup_success.json is defined) and (rmq_cleanup_success.json.routed | default(false) | bool)

    # Mismatch: reachable & banner read, but not equal to target_version
    - name: Build mismatch journal payload
      when: nc_probe.rc == 0 and banner_probe.rc == 0 and not (version_match | bool)
      delegate_to: localhost
      ansible.builtin.set_fact:
        journal_mismatch_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            afterupgrade_check MISMATCH (attempt {{ attempt }}/{{ effective_max_attempts }}):
            Expected='{{ target_version_effective }}' Got='{{ (banner_probe.stdout | default('') | trim) }}'
            Correlation={{ correlation_id }} Original={{ original_emitted_at }}

    - name: Publish mismatch journal to control queue
      when: journal_mismatch_payload is defined
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_mismatch_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_j_mismatch
      changed_when: (rmq_j_mismatch.json is defined) and (rmq_j_mismatch.json.routed | default(false) | bool)

    # SSH error path: TCP OK, but SSH failed
    - name: Build failure journal payload (ssh error) + mark retry
      when: nc_probe.rc == 0 and banner_probe.rc != 0
      delegate_to: localhost
      ansible.builtin.set_fact:
        journal_fail_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            afterupgrade_check FAILED_SSH (attempt {{ attempt }}/{{ effective_max_attempts }}):
            {{ (banner_probe.stderr | default('') | trim) }}
            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'
          _needs_retry: true

    - name: Publish failure journal (ssh error) to control queue
      when: nc_probe.rc == 0 and banner_probe.rc != 0
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_fail_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_j_fail
      changed_when: (rmq_j_fail.json is defined) and (rmq_j_fail.json.routed | default(false) | bool)

    # ---- Retry scheduling (ONLY when we flagged _needs_retry) ----
    - name: Compute next-attempt delay (ms) according to policy
      when: (_needs_retry | default(false)) | bool
      ansible.builtin.set_fact:
        next_attempt: "{{ attempt | int + 1 }}"
        next_delay_sec: >-
          {% if attempt | int == 1 %}
          300
          {% elif attempt | int == 2 %}
          600
          {% else %}
          0
          {% endif %}
        next_delay_ms: "{{ ( (attempt | int == 1) | ternary(300, (attempt | int == 2) | ternary(600, 0)) ) * 1000 }}"

    # If we've reached the cap, send a final “gave up” journal and stop.
    - name: Build final gave-up journal (max attempts reached)
      when: (_needs_retry | default(false)) | bool and (attempt | int) >= (effective_max_attempts | int)
      delegate_to: localhost
      ansible.builtin.set_fact:
        journal_gaveup_payload:
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          task_name: "journal_add"
          task_result: >-
            afterupgrade_check GAVE_UP (attempt {{ attempt }}/{{ effective_max_attempts }}):
            Exhausted attempts. Last error path={{ 'TCP' if nc_probe.rc != 0 else 'SSH' }}.
            Correlation={{ correlation_id }} Original={{ original_emitted_at }} Target='{{ target_version_full }}'

    - name: Publish final gave-up journal
      when: journal_gaveup_payload is defined
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/{{ rmq_exchange | urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
          routing_key: "{{ control_queue }}"
          payload: "{{ journal_gaveup_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_j_gaveup
      changed_when: (rmq_j_gaveup.json is defined) and (rmq_j_gaveup.json.routed | default(false) | bool)

    - name: Stop host after final gave-up
      when: journal_gaveup_payload is defined
      ansible.builtin.meta: end_host

    # Otherwise schedule the next attempt (only if we still have budget)
    - name: Build delayed after-upgrade payload for next attempt
      when: (_needs_retry | default(false)) | bool and (attempt | int) < (effective_max_attempts | int)
      delegate_to: localhost
      ansible.builtin.set_fact:
        delayed_payload:
          task_name: "afterupgrade_check"
          inscope_device: "{{ ansible_hostname | default(inventory_hostname) }}"
          attempt: "{{ next_attempt | int }}"
          max_attempts: "{{ effective_max_attempts | int }}"
          correlation_id: "{{ correlation_id }}"
          original_emitted_at: "{{ original_emitted_at }}"
          target_version: "{{ target_version_full }}"
          current_delay_sec: "{{ next_delay_sec | int }}"
          schema_version: 1

    - name: Publish delayed next attempt to holding exchange (dead-letters to deviceconfig)
      when: delayed_payload is defined
      delegate_to: localhost
      ansible.builtin.uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost | urlencode }}/deviceconfig.holding/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers:
          content-type: "application/json"
        body_format: json
        body:
          properties:
            content_type: "application/json"
            expiration: "{{ (next_delay_ms | int) | string }}"
            correlation_id: "{{ correlation_id }}"
          routing_key: "deviceconfig"
          payload: "{{ delayed_payload | to_json }}"
          payload_encoding: "string"
      register: rmq_pub_next
      changed_when: (rmq_pub_next.json is defined) and (rmq_pub_next.json.routed | default(false) | bool)

    - name: Stop host after TCP/SSH failure (scheduled next or gave-up already)
      when: (_needs_retry | default(false)) | bool
      ansible.builtin.meta: end_host