From 06ffc605f7617dc106c78f08dd2ecfb7334ac4b2 Mon Sep 17 00:00:00 2001 From: pavel Date: Wed, 25 Feb 2026 09:28:23 +0200 Subject: [PATCH] 0928 --- files/netbox-reporter.sh | 125 ++++++++++++++++++++++++++++++++------- 1 file changed, 102 insertions(+), 23 deletions(-) diff --git a/files/netbox-reporter.sh b/files/netbox-reporter.sh index 93e42ac..883dc67 100644 --- a/files/netbox-reporter.sh +++ b/files/netbox-reporter.sh @@ -20,6 +20,17 @@ NB_URL="${NB_URL:-http://netbox.gt-tiso.ikeja.co.za}" # Tip: Prefer setting NB_TOKEN via environment. Leaving default blank avoids accidental leaks. NB_TOKEN="${NB_TOKEN:-}" +# --- Optional performance knobs --- +# NB_PREREAD=1 enables extra GETs to print pre-change values (more load on NetBox). +NB_PREREAD="${NB_PREREAD:-0}" + +# Device ID cache (per worker process) +# Cache entries are considered fresh for 60 minutes. After that, they may be used up to an +# additional random 3-10 minutes ("stale window") before a refresh is attempted. +NB_IDCACHE_TTL_SECS="${NB_IDCACHE_TTL_SECS:-3600}" +NB_IDCACHE_STALE_MIN_SECS="${NB_IDCACHE_STALE_MIN_SECS:-180}" +NB_IDCACHE_STALE_MAX_SECS="${NB_IDCACHE_STALE_MAX_SECS:-600}" + log() { echo "[netbox-reporter] $*"; } warn(){ echo "[netbox-reporter][WARN] $*" >&2; } err() { echo "[netbox-reporter][ERROR] $*" >&2; } @@ -33,6 +44,54 @@ fi # URL-encode for RabbitMQ HTTP API paths urlenc() { printf '%s' "$1" | sed -e 's, ,%20,g' -e 's,/,%2F,g'; } +# --- helpers --- +now_epoch() { date +%s; } +rand_between() { + local min="$1" max="$2" + # inclusive range + echo $(( min + (RANDOM % (max - min + 1)) )) +} + +# Device ID cache (associative arrays require bash 4+) +declare -A NB_DEV_ID_CACHE=() +declare -A NB_DEV_ID_CACHE_EXPIRES=() +declare -A NB_DEV_ID_CACHE_STALE_UNTIL=() + +nb_get_device_id_cached() { + local name="$1" + local now exp stale id jitter + + now="$(now_epoch)" + id="${NB_DEV_ID_CACHE[$name]:-}" + exp="${NB_DEV_ID_CACHE_EXPIRES[$name]:-0}" + stale="${NB_DEV_ID_CACHE_STALE_UNTIL[$name]:-0}" + + # Fresh cache hit + if [[ -n "$id" && "$now" -lt "$exp" ]]; then + printf '%s' "$id" + return 0 + fi + + # Stale-but-usable cache hit (during jitter window): return cached ID, don't refresh yet + if [[ -n "$id" && "$now" -ge "$exp" && "$now" -lt "$stale" ]]; then + printf '%s' "$id" + return 0 + fi + + # Cache miss or refresh time: refresh via NetBox + id="$(nb_find_device_id "$name")" || true + if [[ -n "$id" ]]; then + NB_DEV_ID_CACHE["$name"]="$id" + NB_DEV_ID_CACHE_EXPIRES["$name"]=$(( now + NB_IDCACHE_TTL_SECS )) + jitter="$(rand_between "$NB_IDCACHE_STALE_MIN_SECS" "$NB_IDCACHE_STALE_MAX_SECS")" + NB_DEV_ID_CACHE_STALE_UNTIL["$name"]=$(( now + NB_IDCACHE_TTL_SECS + jitter )) + printf '%s' "$id" + return 0 + fi + + return 1 +} + # Simple RabbitMQ HTTP call rmq_api() { local method="$1" path="$2" data="${3:-}" @@ -51,7 +110,7 @@ nb_find_device_id() { resp="$(curl -sS \ -H "Authorization: Token $NB_TOKEN" \ -H "Accept: application/json" \ - "$NB_URL/api/dcim/devices/?name=$(printf '%s' "$name" | jq -sRr @uri)&limit=1")" || return 1 + "$NB_URL/api/dcim/devices/?name=$(printf '%s' "$name" | jq -sRr @uri)&limit=1&fields=id")" || return 1 jq -r '.results[0].id // empty' <<<"$resp" } @@ -62,7 +121,7 @@ nb_read_custom_field() { resp="$(curl -sS \ -H "Authorization: Token $NB_TOKEN" \ -H "Accept: application/json" \ - "$NB_URL/api/dcim/devices/$dev_id/")" || return 1 + "$NB_URL/api/dcim/devices/$dev_id/?fields=custom_fields")" || return 1 jq -r --arg k "$key" ' (.custom_fields[$k] // empty) | (if type=="object" or type=="array" then tojson else . end) @@ -73,11 +132,15 @@ nb_read_custom_field() { nb_patch_update_wo_restart() { local dev_id="$1" update_progress="$2" updating_to="$3" - # Pre-read current values and print + # Pre-read current values and print (optional; enable with NB_PREREAD=1) local prev_p prev_u - prev_p="$(nb_read_custom_field "$dev_id" "update_progress" || true)" - prev_u="$(nb_read_custom_field "$dev_id" "updating_to" || true)" - echo ">>> Pre-change (id=$dev_id): update_progress='${prev_p:-}'; updating_to='${prev_u:-}'" + if [[ "${NB_PREREAD}" == "1" ]]; then + prev_p="$(nb_read_custom_field "$dev_id" "update_progress" || true)" + prev_u="$(nb_read_custom_field "$dev_id" "updating_to" || true)" + echo ">>> Pre-change (id=$dev_id): update_progress='${prev_p:-}'; updating_to='${prev_u:-}'" + else + echo ">>> Pre-change (id=$dev_id): update_progress=''; updating_to=''" + fi local body body="$(jq -n --arg p "$update_progress" --arg u "$updating_to" \ @@ -103,10 +166,14 @@ nb_patch_update_wo_restart() { nb_patch_wifidebug() { local dev_id="$1" value="$2" - # Pre-read current value and print + # Pre-read current value and print (optional; enable with NB_PREREAD=1) local prev - prev="$(nb_read_custom_field "$dev_id" "wifidebug" || true)" - echo ">>> Pre-change (id=$dev_id): wifidebug='${prev:-}'" + if [[ "${NB_PREREAD}" == "1" ]]; then + prev="$(nb_read_custom_field "$dev_id" "wifidebug" || true)" + echo ">>> Pre-change (id=$dev_id): wifidebug='${prev:-}'" + else + echo ">>> Pre-change (id=$dev_id): wifidebug=''" + fi local body body="$(jq -n --arg v "$value" '{custom_fields: {wifidebug: $v}}')" @@ -178,10 +245,14 @@ nb_add_journal_raw() { nb_set_multiple_ssids_yes() { local dev_id="$1" - # Pre-read current value and print + # Pre-read current value and print (optional; enable with NB_PREREAD=1) local prev - prev="$(nb_read_custom_field "$dev_id" "multiple_ssids" || true)" - echo ">>> Pre-change (id=$dev_id): multiple_ssids='${prev:-}'" + if [[ "${NB_PREREAD}" == "1" ]]; then + prev="$(nb_read_custom_field "$dev_id" "multiple_ssids" || true)" + echo ">>> Pre-change (id=$dev_id): multiple_ssids='${prev:-}'" + else + echo ">>> Pre-change (id=$dev_id): multiple_ssids=''" + fi local body='{"custom_fields":{"multiple_ssids":"yes"}}' @@ -241,7 +312,7 @@ nb_add_tag_by_slug() { dev_detail="$(curl -sS \ -H "Authorization: Token $NB_TOKEN" \ -H "Accept: application/json" \ - "$NB_URL/api/dcim/devices/$dev_id/")" + "$NB_URL/api/dcim/devices/$dev_id/?fields=tags")" existing_objs_json="$( jq -c ' (.tags // []) as $t | @@ -294,7 +365,7 @@ nb_remove_tag_by_slug() { dev_detail="$(curl -sS \ -H "Authorization: Token $NB_TOKEN" \ -H "Accept: application/json" \ - "$NB_URL/api/dcim/devices/$dev_id/")" || return 1 + "$NB_URL/api/dcim/devices/$dev_id/?fields=tags")" || return 1 # Normalize tags to objects: [{name, slug?}] existing_objs_json="$( @@ -353,10 +424,14 @@ nb_remove_tag_by_slug() { nb_clear_update_progress() { local dev_id="$1" - # Pre-read current value and print + # Pre-read current value and print (optional; enable with NB_PREREAD=1) local prev - prev="$(nb_read_custom_field "$dev_id" "update_progress" || true)" - echo ">>> Pre-change (id=$dev_id): update_progress='${prev:-}'" + if [[ "${NB_PREREAD}" == "1" ]]; then + prev="$(nb_read_custom_field "$dev_id" "update_progress" || true)" + echo ">>> Pre-change (id=$dev_id): update_progress='${prev:-}'" + else + echo ">>> Pre-change (id=$dev_id): update_progress=''" + fi local body='{"custom_fields":{"update_progress":null}}' local code @@ -378,10 +453,14 @@ nb_clear_update_progress() { nb_patch_custom_field() { local dev_id="$1" key="$2" value="$3" - # Pre-read current value and print + # Pre-read current value and print (optional; enable with NB_PREREAD=1) local prev - prev="$(nb_read_custom_field "$dev_id" "$key" || true)" - echo ">>> Pre-change (id=$dev_id): ${key}='${prev:-}'" + if [[ "${NB_PREREAD}" == "1" ]]; then + prev="$(nb_read_custom_field "$dev_id" "$key" || true)" + echo ">>> Pre-change (id=$dev_id): ${key}='${prev:-}'" + else + echo ">>> Pre-change (id=$dev_id): ${key}=''" + fi # Build {"custom_fields": { "": "" }} local body @@ -423,7 +502,7 @@ handle_payload() { fi local dev_id - dev_id="$(nb_find_device_id "$device")" + dev_id="$(nb_get_device_id_cached "$device")" if [[ -z "$dev_id" ]]; then err "Device '$device' not found in NetBox" return 0 @@ -509,7 +588,7 @@ handle_payload() { echo ">>> Device '$device' (id=$dev_id) updated:" echo " tag removed (slug) = '$task_result'" fi - ;; + ;; *) log "Ignoring unsupported task_name='$task' (no-op)" ;; @@ -546,4 +625,4 @@ while :; do warn "Skipping non-JSON payload" fi done -done +done \ No newline at end of file