# sot-updater.yml — Read fw on Dev1 + Dev2, publish to NetBox via Rabbit (telemetry mode, no upgrade scheduling)

- name: Read fw on Dev1 + Dev2, publish NetBox custom fields (full base, AIRPINGs, soft-fail telemetry, no upgrade scheduling)
  hosts: all
  gather_facts: no

  vars:
    pathprefix: "PATH=/sbin:/usr/sbin:/bin:/usr/bin:$PATH; "
    dev1_user: "root"
    dev1_pass: "wavewave"
    ssh_timeout: 30

    dev2_host: "192.168.1.1"
    dev2_port: 22
    dev2_ssh_user: "root"
    dev2_passfiles: [ "basicpass", "basicpass2" ]

    dev2_side_ip: "192.168.1.11/24"
    dev1_iface: "br-wan"
    arping_iface: "eth0"
    dev2_side_ip_addr: "{{ dev2_side_ip.split('/')[0] }}"

    ssh_opts_common: >-
      -o PreferredAuthentications=password
      -o PubkeyAuthentication=no
      -o StrictHostKeyChecking=no
      -o UserKnownHostsFile=/dev/null
      -o NumberOfPasswordPrompts=1
      -o ConnectTimeout=30

    rmq_host: "{{ lookup('env','RMQ_HOST') | default('10.210.12.2', true) }}"
    rmq_port: "{{ lookup('env','RMQ_PORT') | default('15672', true) }}"
    rmq_user: "{{ lookup('env','RMQ_USER') | default('admin', true) }}"
    rmq_pass: "{{ lookup('env','RMQ_PASS') | default('change_me', true) }}"
    rmq_vhost: "{{ lookup('env','RMQ_VHOST') | default('app', true) }}"
    rmq_exchange: "{{ lookup('env','RMQ_EXCHANGE') | default('controls', true) }}"
    control_queue: "{{ lookup('env','CONTROLQUEUE') | default('queue_controls', true) }}"

    # --- Retry attempt tracking (default 0 if missing) ---
    requeue_attempt: "{{ (lookup('env','REQUEUE_ATTEMPT') | default('0', true)) | int }}"

  pre_tasks:
    - name: "NB preflight | Verify script exists"
      delegate_to: localhost
      ansible.builtin.stat:
        path: "/opt/containers/ansible-worker/app/nb_onedevice_update.py"
      register: nb_script

    - name: "NB preflight | Abort softly if script missing"
      when: not nb_script.stat.exists
      delegate_to: localhost
      ansible.builtin.debug:
        msg:
          - "NB preflight skipped: /opt/containers/ansible-worker/app/nb_onedevice_update.py not found."
          - "Tip: adjust chdir/path or script name."

    - name: "NB preflight | Run nb_onedevice_update.py for {{ inventory_hostname }}"
      when: nb_script.stat.exists
      delegate_to: localhost
      environment:
        PYTHONUNBUFFERED: "1"
      args:
        chdir: "/opt/containers/ansible-worker/app"
        executable: /bin/bash
      shell: |
        set -o pipefail
        python3 -u nb_onedevice_update.py "{{ inventory_hostname }}" --chatty 2>&1
      register: nb_preflight
      changed_when: false
      failed_when: false

    - name: "NB preflight | Show results"
      when: nb_script.stat.exists
      delegate_to: localhost
      debug:
        msg:
          - "rc={{ nb_preflight.rc }}"
          - "stdout_lines:"
          - "{{ (nb_preflight.stdout_lines | default(['<no stdout>'])) }}"
          - "stderr_lines:"
          - "{{ (nb_preflight.stderr_lines | default(['<no stderr>'])) }}"
          - "raw stdout (joined): {{ nb_preflight.stdout | default('') | trim }}"

    # --- Detect cloud failure (rc==3 or FAIL Cloud line) ---
    - name: "NB preflight | Detect cloud failure"
      when: nb_script.stat.exists
      delegate_to: localhost
      vars:
        _lines: >-
          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
        _cloud_fail_line: >-
          {{ (_lines | select('match', '^FAIL\\s+Cloud\\b') | list | last | default('')) }}
        _rc_is_3: "{{ (nb_preflight.rc | default(1)) | int == 3 }}"
      set_fact:
        cloud_fail_line: "{{ _cloud_fail_line }}"
        cloud_bad: "{{ _rc_is_3 or (( _cloud_fail_line | length ) > 0) }}"

    # --- Extract upgrade_cmd line if any ---
    - name: "NB preflight | Extract upgrade_cmd line"
      when: nb_script.stat.exists
      delegate_to: localhost
      set_fact:
        nb_upgrade_line: >-
          {{
            (
              nb_preflight.stdout_lines | default([]) | map('regex_replace','\r','') | map('trim')
              | select('match', '^NB:\\s*upgrade_cmd\\s*=')
              | list | first
            ) | default('')
          }}

    - name: "NB preflight | Parse upgrade_cmd value"
      when: nb_upgrade_line | length > 0
      delegate_to: localhost
      shell: |
        printf '%s\n' "{{ nb_upgrade_line }}" | awk -F'=' '{print $2}' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'
      args: { executable: /bin/bash }
      register: up_cmd_sh
      changed_when: false

    - name: "NB preflight | Set parsed upgrade_cmd"
      when: nb_upgrade_line | length > 0
      delegate_to: localhost
      set_fact:
        nb_upgrade_cmd: "{{ up_cmd_sh.stdout | default('') | trim }}"

    - name: "NB preflight | Debug parsed upgrade_cmd"
      when: nb_upgrade_line | length > 0
      delegate_to: localhost
      debug:
        msg:
          - "upgrade_cmd='{{ nb_upgrade_cmd }}' (len={{ nb_upgrade_cmd|length }})"

    # --- Parse OK line & detect IP change ---
    - name: "NB preflight | Parse OK line"
      when: nb_script.stat.exists
      delegate_to: localhost
      vars:
        nb_lines: >-
          {{ (nb_preflight.stdout | default('') | regex_replace('\r','')) | split('\n') | map('trim') | list }}
        nb_ok_line: >-
          {{ (nb_lines | select('match', '^OK\\s+') | list | last | default('')) }}
        nb_tokens: >-
          {{ (nb_ok_line | regex_replace('^OK\\s+', '')).split() }}
        nb_kv: >-
          {{
            dict(
              nb_tokens
              | select('match', '^[a-zA-Z_]+=')
              | map('split', '=', 1)
              | map('list')
            )
          }}
      set_fact:
        nb_ok: "{{ (nb_preflight.rc|default(1)) == 0 and (nb_ok_line|length)>0 }}"
        nb_dev: "{{ (nb_tokens|first|default('')) if (nb_tokens|length>0) else '' }}"
        nb_ip: "{{ nb_kv.get('ip','') }}"
        nb_fw: "{{ nb_kv.get('fw','') }}"
        nb_node: "{{ nb_kv.get('node','') }}"
        nb_sector: "{{ nb_kv.get('sector','') }}"
        nb_small: "{{ nb_kv.get('small','') }}"
        nb_ok_line: "{{ nb_ok_line }}"

    - name: "NB preflight | Detect IP change"
      when: nb_script.stat.exists
      delegate_to: localhost
      vars:
        out: "{{ (nb_preflight.stdout | default('') | regex_replace('\r','')) }}"
        reason: >-
          {%- if 'IP: moving ' in out -%}moving
          {%- elif 'IP: create new ' in out -%}create new
          {%- elif 'IP: pruning stale ' in out -%}pruning stale
          {%- else -%}none{%- endif -%}
      set_fact:
        nb_ip_changed: "{{ reason != 'none' }}"
        nb_change_reason: "{{ reason }}"

    - name: "NB preflight | Verdict"
      when: nb_script.stat.exists
      delegate_to: localhost
      debug:
        msg:
          - "Preflight verdict:"
          - "Cloud fail: {{ cloud_bad|default(false) }}"
          - "IP changed: {{ nb_ip_changed|default(false) }}"
          - "Reason: {{ nb_change_reason|default('none') }}"

    # --- Requeue control: only 3 attempts, 300s apart ---
    - name: "NB preflight | Publish delayed requeue (300s) if needed"
      when:
        - nb_script.stat.exists
        - (cloud_bad | default(false)) or (nb_ip_changed | default(false))
        - (requeue_attempt | int) < 3
      delegate_to: localhost
      uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/app/deviceconfig.delayed/publish"
        method: POST
        user: "admin"
        password: "change_me"
        force_basic_auth: true
        status_code: 200
        return_content: yes
        headers: { content-type: "application/json" }
        body_format: json
        body:
          properties:
            content_type: "application/json"
            headers: { x-delay: 300000 }   # 300 seconds = 5 minutes
          routing_key: "deviceconfig"
          payload: "{{ {
            'inscope_device': (ansible_hostname | default(inventory_hostname)),
            'task_name': 'sot-updater',
            'attempt': ((requeue_attempt | int) + 1)
          } | to_json }}"
          payload_encoding: "string"
      register: rmq_requeue
      changed_when: false
      failed_when: false

    - name: "NB preflight | Log requeue publish response"
      when:
        - nb_script.stat.exists
        - ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
        - (requeue_attempt | int) < 3
      delegate_to: localhost
      debug:
        msg:
          - "Requeue published (attempt {{ requeue_attempt+1 }})"
          - "HTTP: {{ rmq_requeue.status|default('n/a') }}"
          - "JSON: {{ rmq_requeue.json|default('<none>') }}"

    - name: "NB preflight | Stop host after requeue"
      when:
        - nb_script.stat.exists
        - ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
      meta: end_host

    - name: "NB preflight | Pause 1s if OK"
      when:
        - nb_script.stat.exists
        - not ((cloud_bad | default(false)) or (nb_ip_changed | default(false)))
      delegate_to: localhost
      pause:
        seconds: 1

  tasks:
    - name: "⚙️ Start | Dev1 banner → tunnel/auth → Dev2 firmux (telemetry mode)"
      debug:
        msg:
          - "Device: {{ inventory_hostname }}"
          - "Mode: report-only (soft-fail telemetry)"

    # ----------------------- Temp IP on DEV1 -----------------------
    - name: Add temporary IP on DEV1
      raw: "{{ pathprefix }} ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}"
      register: add_ip
      failed_when: false
      ignore_errors: true

    - name: Add temporary IP on DEV1 (tolerate 'File exists')
      raw: >
        {{ pathprefix }}
        ip a add {{ dev2_side_ip }} dev {{ dev1_iface }}
      register: add_ip
      changed_when: add_ip.rc == 0
      failed_when: >
        add_ip.rc != 0
        and ('File exists' not in (add_ip.stdout|default('')))
        and ('File exists' not in (add_ip.stderr|default('')))

    - name: Debug temp IP result
      debug:
        msg:
          - "add_ip.rc={{ add_ip.rc|default('') }}"
          - "add_ip.out={{ add_ip.stdout|default('')|trim }}"
          - "add_ip.err={{ add_ip.stderr|default('')|trim }}"

    - name: Discover DEV2 MAC via bridge fdb
      raw: >
        {{ pathprefix }}
        bridge fdb show {{ dev1_iface }} | grep eth0 | grep -v permanent | grep master | awk '{print $1}' | head -n1
      register: dev2_mac_scan
      changed_when: false
      failed_when: false

    - name: Capture discovered MAC
      set_fact:
        dev2_mac: "{{ dev2_mac_scan.stdout|default('')|trim }}"

    - name: Clear existing ARP entry
      raw: >
        {{ pathprefix }}
        ip neigh del {{ dev2_host }} dev {{ dev1_iface }} 2>/dev/null || true
      changed_when: false
      failed_when: false

    - name: Add static ARP if MAC found
      when: dev2_mac|length > 0
      raw: >
        {{ pathprefix }}
        ip neigh add {{ dev2_host }} lladdr {{ dev2_mac }} dev {{ dev1_iface }} nud permanent
      register: dev2_arp_add
      changed_when: dev2_arp_add.rc == 0
      failed_when: >
        dev2_arp_add.rc != 0
        and ('File exists' not in (dev2_arp_add.stdout|default('')))
        and ('File exists' not in (dev2_arp_add.stderr|default('')))

    - name: Debug ARP summary
      debug:
        msg:
          - "dev2_mac={{ dev2_mac|default('unset') }}"
          - "arp_add.rc={{ dev2_arp_add.rc|default('') }}"
          - "arp_add.out={{ dev2_arp_add.stdout|default('')|trim }}"
          - "arp_add.err={{ dev2_arp_add.stderr|default('')|trim }}"

    - name: Note skipping static ARP
      when: dev2_mac|length == 0
      debug:
        msg: "No MAC discovered; skipping static ARP add."

    - name: Refresh ARP
      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
      failed_when: false
      ignore_errors: true

    # ----------------------- Tunnel setup -----------------------
    - name: Pick free local port
      delegate_to: localhost
      shell: |
        for i in $(seq 1 50); do
          p="$(shuf -i 20000-39999 -n 1)"
          ss -ltn | awk '{print $4}' | grep -qE "(:|\.)${p}$" || { echo $p; exit 0; }
        done
        exit 1
      register: pick_port
      failed_when: false

    - name: Create control dir
      delegate_to: localhost
      shell: "mktemp -d"
      register: mktemp_dir
      failed_when: false

    - name: Set tunnel facts
      delegate_to: localhost
      set_fact:
        _local_port: "{{ pick_port.stdout|default('')|trim }}"
        _ctrl_dir: "{{ mktemp_dir.stdout|default('')|trim }}"
        _ctrl_sock: "{{ mktemp_dir.stdout|default('')|trim }}/ssh_tunnel_ctl"

    - name: Start SSH tunnel
      delegate_to: localhost
      shell: |
        set -e
        USER="{{ dev1_user }}"
        HOST="{{ ansible_host|default(inventory_hostname) }}"
        sshpass -p '{{ dev1_pass }}' ssh -f -N {{ ssh_opts_common }} \
          -M -S "{{ _ctrl_sock }}" \
          -L "127.0.0.1:{{ _local_port }}:{{ dev2_host }}:{{ dev2_port }}" \
          "${USER}@${HOST}"
      args: { executable: /bin/bash }
      failed_when: false
      ignore_errors: true

    - name: Probe tunnel
      delegate_to: localhost
      shell: "nc -z -w5 127.0.0.1 {{ _local_port }}"
      register: nc_probe
      failed_when: false
      ignore_errors: true

    - name: Set tunnel_ok
      delegate_to: localhost
      set_fact:
        tunnel_ok: "{{ nc_probe.rc == 0 }}"

    # ----------------------- Dev1 banner -----------------------
    - name: Dev1 | Probe banner
      delegate_to: localhost
      shell: |
        sshpass -p '{{ dev1_pass }}' ssh {{ ssh_opts_common }} \
        "{{ dev1_user }}@{{ ansible_host|default(inventory_hostname) }}" \
        "cat /etc/banner | grep -i rev | head -n1"
      register: dev1_banner
      failed_when: false
      ignore_errors: true

    - name: Dev1 | Normalize banner
      delegate_to: localhost
      shell: |
        printf '%s\n' "{{ dev1_banner.stdout|trim }}" \
        | awk -F '|' '{print $1}' \
        | sed -E 's/[[:space:]]+rev[[:space:]]+/-r/' \
        | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+-r[0-9]+' || true
      register: dev1_fw_clean_cmd
      changed_when: false

    - name: Dev1 | Set final fw string
      delegate_to: localhost
      set_fact:
        dev1_fw_clean: "{{ dev1_fw_clean_cmd.stdout|trim }}"

    - name: Publish Dev1 fw_version
      delegate_to: localhost
      uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers: { content-type: "application/json" }
        body_format: json
        body:
          properties: { content_type: "application/json" }
          routing_key: "{{ control_queue }}"
          payload: "{{ {
            'inscope_device': (ansible_hostname|default(inventory_hostname)),
            'task_name': 'custom_field_set',
            'task_add1': 'fw_version',
            'task_result': (dev1_fw_clean if (dev1_fw_clean|length>0) else 'unavailable')
          } | to_json }}"
          payload_encoding: "string"
      changed_when: false

    # ----------------------- Dev2 auth + firmux -----------------------
    - name: Try DEV2 login
      when: tunnel_ok|default(false)
      delegate_to: localhost
      shell: |
        for f in basicpass basicpass2; do
          PORT="{{ _local_port }}"
          if sshpass -f "$f" ssh -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
               -p "$PORT" root@127.0.0.1 "echo ok" >/dev/null 2>&1; then
            echo "$f"; exit 0
          fi
        done
        exit 1
      register: dev2_auth
      failed_when: false
      ignore_errors: true

    - name: Set dev2_passfile_used
      delegate_to: localhost
      set_fact:
        dev2_passfile_used: "{{ (dev2_auth.rc == 0)|ternary(dev2_auth.stdout|trim,'NONE') }}"

    - name: Dev2 | Read firmux
      when: tunnel_ok|default(false) and dev2_passfile_used != 'NONE'
      delegate_to: localhost
      shell: |
        PORT="{{ _local_port }}"
        sshpass -f "{{ dev2_passfile_used }}" ssh \
          -o StrictHostKeyChecking=no -o PubkeyAuthentication=no \
          -o ConnectTimeout=15 \
          -p "$PORT" "{{ dev2_ssh_user }}@127.0.0.1" \
          "cat /usr/lib/release/firmux 2>/dev/null || true"
      register: dev2_firmux
      failed_when: false
      ignore_errors: true

    - name: Dev2 | Extract firmware version line
      delegate_to: localhost
      run_once: true
      set_fact:
        indoor_fw_norm: >-
          {{
            (
              (dev2_firmux.stdout|default('')|regex_replace('\r',''))|split('\n')
              | map('trim') | select('truthy') | list
            ) | select('match','(?i).*\\brev\\s*[0-9]+.*')
              | list | first | default(
                  ((dev2_firmux.stdout|default('')|regex_replace('\r',''))
                   | split('\n')|map('trim')|select('truthy')|list|first|default(''))
                ) | trim
          }}

    - name: Publish Dev2 indoor_fwver
      delegate_to: localhost
      uri:
        url: "http://{{ rmq_host }}:{{ rmq_port }}/api/exchanges/{{ rmq_vhost|urlencode }}/{{ rmq_exchange|urlencode }}/publish"
        method: POST
        user: "{{ rmq_user }}"
        password: "{{ rmq_pass }}"
        force_basic_auth: true
        status_code: 200
        headers: { content-type: "application/json" }
        body_format: json
        body:
          properties: { content_type: "application/json" }
          routing_key: "{{ control_queue }}"
          payload: "{{ {
            'inscope_device': (ansible_hostname|default(inventory_hostname)),
            'task_name': 'custom_field_set',
            'task_add1': 'indoor_fwver',
            'task_result': (indoor_fw_norm if indoor_fw_norm|length>0 else 'unavailable')
          } | to_json }}"
          payload_encoding: "string"
      changed_when: false

  post_tasks:
    - name: "✅ Cleanup start"
      debug:
        msg: "Closing tunnel and removing temporary IP"

    - name: Refresh ARP (cleanup)
      raw: "{{ pathprefix }} arping -U -I {{ arping_iface }} {{ dev2_side_ip_addr }} -c 3"
      failed_when: false
      ignore_errors: true

    - name: Close SSH ControlMaster
      delegate_to: localhost
      shell: "ssh -S '{{ _ctrl_sock|default('/dev/null') }}' -O exit 2>/dev/null || true"
      failed_when: false
      ignore_errors: true

    - name: Remove control dir
      delegate_to: localhost
      file:
        path: "{{ _ctrl_dir|default('/tmp/none') }}"
        state: absent
      ignore_errors: true

    - name: Remove temporary IP
      raw: "{{ pathprefix }} ip a del {{ dev2_side_ip }} dev {{ dev1_iface }}"
      failed_when: false
      ignore_errors: true

    - name: "✅ Completed | Device processed"
      debug:
        msg:
          - "Device: {{ inventory_hostname }}"
          - "Status: DONE (telemetry only)"