#!/usr/bin/env bash
# ============================================================================
# TrakopLens RAG reindex SCHEDULER — the nightly rebuild, owned by the stack.
#
# WHY THIS EXISTS
#   The reindex work was always containerised (the `rag-reindex` service), but
#   the SCHEDULE was not: it lived in a host crontab pointing at
#       /var/www/html/trakop-web/ops/cron_trakoplens_rag_reindex.sh
#   That made the nightly rebuild depend on a path outside the deployment, and
#   the moment the checkout was renamed the job stopped running — silently,
#   because a cron whose command is missing writes nothing anyone reads. The two
#   corpora that only a schedule can maintain then freeze:
#     schema — the table/column corpus QueryPlanner resolves tables from, so a
#              newly migrated table stays invisible to planning.
#     qa     — the 👍-validated question/answer loop, i.e. everything the
#              assistant learns from its own approved answers.
#   Neither failure has a runtime symptom: every retriever fails OPEN, so the
#   assistant just quietly answers from a corpus that stopped being updated.
#
# WHAT IT DOES
#   Sleeps until the next RAG_REINDEX_AT (local time, TZ-aware), runs the SAME
#   /usr/local/bin/trakop-rag-reindex the one-shot service runs — same image,
#   same mounts, same env, same validation and exit codes — then records the
#   outcome and sleeps again. It is a scheduler and nothing else: no second
#   implementation of the reindex to drift from the first.
#
#   It does NOT shell out to `docker` to launch a sibling container, which would
#   mean mounting the Docker socket — handing full root-equivalent control of
#   the host to a long-running service, to save writing this loop.
#
# READINESS, NOT LIVENESS
#   The heartbeat proves the loop is alive; the status file records the last
#   completed run. A FAILED nightly run leaves the container UNHEALTHY on
#   purpose: a stale index is invisible everywhere else, so `docker compose ps`
#   is made the one place it shows. Nothing depends on this service, so an
#   unhealthy scheduler is an alarm, never an outage.
#
#     reindex-scheduler.sh            supervise: run at RAG_REINDEX_AT, forever
#     reindex-scheduler.sh --once     run now, once, and exit with the run's code
# ============================================================================
set -uo pipefail

APP_ROOT=/var/www/html/trakop-web/code
STATE_DIR=/var/run/trakop
HEARTBEAT="$STATE_DIR/reindex-scheduler.heartbeat"
STATUS="$STATE_DIR/reindex-scheduler.status"
# Preserved deliberately: this is the file the host cron appended to for months,
# so the operational history stays in one place across the cutover. Best effort —
# it is a bind mount owned by a host uid and this entrypoint does not run the web
# container's chmod step, so an unwritable mount must not stop the schedule.
HOST_LOG="$APP_ROOT/logs/trakoplens-reindex.log"

AT="${RAG_REINDEX_AT:-03:20}"
RUNNER=/usr/local/bin/trakop-rag-reindex

log() {
    local line
    line="[reindex-scheduler] $(date '+%Y-%m-%d %H:%M:%S %Z') $*"
    printf '%s\n' "$line"
    [ -w "$HOST_LOG" ] && printf '%s\n' "$line" >> "$HOST_LOG" 2>/dev/null
    return 0
}

mkdir -p "$STATE_DIR" 2>/dev/null || true

case "$AT" in
    [0-2][0-9]:[0-5][0-9]) : ;;
    *) log "FATAL: RAG_REINDEX_AT must be HH:MM (24h), got '${AT}'"; exit 1 ;;
esac
[ -x "$RUNNER" ] || { log "FATAL: ${RUNNER} is missing from this image."; exit 1; }

# One reindex, recording the outcome the healthcheck reads. Never `exit`s on a
# failed run: the scheduler's job is to try again tomorrow, and a crash-loop
# would replace one silent failure with a noisier one that still never runs.
run_once() {
    local rc
    log "starting reindex (corpus=${RAG_CORPUS:-all} backend=${RAG_BACKEND:-auto})"
    # flock so a scheduled run and a manual `docker compose run --rm rag-reindex`
    # cannot embed the same corpus concurrently. The lock lives on the bind-mounted
    # tree, so it is the same inode in both containers. -w 0 with -E 99: if the other
    # holder is a deliberate manual run, skip this window rather than queue a second
    # 25-minute job behind it — the corpus it is building is the same one. 99 is
    # distinguishable from any exit code the runner itself produces, so "someone else
    # is already doing it" is never recorded as a failure.
    if [ -w "$APP_ROOT/logs" ]; then
        flock -w 0 -E 99 "$APP_ROOT/logs/.rag-reindex.lock" "$RUNNER"
        rc=$?
    else
        "$RUNNER"
        rc=$?
    fi
    if [ "$rc" -eq 99 ]; then
        log "skipped: another reindex holds the lock (a manual run is in progress). The corpus it builds is the same one."
        return 0
    fi
    if [ "$rc" -eq 0 ]; then
        printf 'ok %s\n' "$(date -u +%FT%TZ)" > "$STATUS" 2>/dev/null
        log "reindex finished OK"
    else
        printf 'fail rc=%s %s\n' "$rc" "$(date -u +%FT%TZ)" > "$STATUS" 2>/dev/null
        log "reindex FAILED rc=${rc} — the index is STALE, not broken: the previous one is still in place. See the lines above for the reason."
    fi
    return "$rc"
}

if [ "${1:-}" = "--once" ]; then
    date -u +%FT%TZ > "$HEARTBEAT" 2>/dev/null
    run_once
    exit $?
fi

log "scheduler up; nightly reindex at ${AT} (TZ=$(date '+%Z'), now $(date '+%H:%M'))"
if [ "${RAG_REINDEX_ON_BOOT:-0}" = "1" ]; then
    log "RAG_REINDEX_ON_BOOT=1 — running once at startup"
    run_once || true
else
    # No status yet = never run in this container's life. Report healthy rather
    # than unhealthy: "has not reached 03:20 yet" is not a failure.
    [ -s "$STATUS" ] || printf 'pending %s\n' "$(date -u +%FT%TZ)" > "$STATUS" 2>/dev/null
fi

while :; do
    now="$(date +%s)"
    target="$(date -d "today ${AT}" +%s 2>/dev/null)" || target=""
    if [ -z "$target" ]; then
        log "FATAL: this image's date(1) cannot parse \"today ${AT}\"."
        exit 1
    fi
    [ "$target" -le "$now" ] && target="$(date -d "tomorrow ${AT}" +%s)"
    remaining=$((target - now))
    log "next run in $((remaining / 3600))h $(((remaining % 3600) / 60))m"

    # Sleep in short chunks so the heartbeat stays fresh and a `docker stop`
    # is not waiting on a multi-hour sleep to return.
    while [ "$remaining" -gt 0 ]; do
        date -u +%FT%TZ > "$HEARTBEAT" 2>/dev/null
        chunk=60; [ "$remaining" -lt 60 ] && chunk="$remaining"
        sleep "$chunk"
        remaining=$((remaining - chunk))
    done

    date -u +%FT%TZ > "$HEARTBEAT" 2>/dev/null
    run_once || true
    # Past the target now; the next loop computes tomorrow. Guard against a run
    # that finished inside the same minute triggering twice.
    sleep 61
done