#!/usr/bin/env bash
# ============================================================================
# rag-reindex — one-shot service. `docker compose run --rm rag-reindex`
#
# WHY THIS IS A SERVICE AND NOT A DOCUMENTED COMMAND:
#   The RAG index is derived from the code and from the KB artefact, so a deploy
#   that adds a tool, edits a description or ships a new snapshot leaves it
#   stale. A stale index is INVISIBLE: every retriever fails open, so the
#   assistant keeps answering — just without KB grounding, without tool
#   narrowing and without walkthrough routing. That exact staleness has twice
#   gone unnoticed for days. A step that can be forgotten will be; a service
#   with an exit code cannot be.
#
# WHAT IT DOES
#   1. converges the pgvector schema (idempotent; covers pre-existing volumes
#      that never ran /docker-entrypoint-initdb.d)
#   2. asserts the embedding dimension against the stored vector width
#   3. runs every indexer via RagReindexShell
#   4. VALIDATES the resulting row counts per corpus and exits non-zero if a
#      corpus that must not be empty is empty
#   5. prints the corpus version so a QA result can be attributed to a snapshot
#
# EXIT CODES
#   0  reindex completed and all corpus minimums satisfied
#   1  preflight failed (schema, dimension, or store unreachable)
#   2  the reindex shell itself failed
#   3  reindex ran but a required corpus is below its minimum
# ============================================================================
set -uo pipefail

APP_ROOT=/var/www/html/trakop-web/code
CORPUS="${RAG_CORPUS:-all}"
# WHICH INDEX to build. Default `auto` = pgvector, exactly as this service has always
# behaved. Set `all` (= pgvector + the MySQL lexical index) on a host that is ALSO started
# without Docker: TrakopLens then falls back to an index in the application's own MySQL,
# and an index nobody rebuilds is the same invisible staleness this service exists to
# prevent — one mode answering from a corpus that stopped being updated, with no symptom
# until the containers are down. `all` adds seconds, not a second embedding pass. The
# container reaches MySQL through the same mounted socket the app uses, so no extra wiring
# is needed. See ops/trakoplens-host-mode.md.
BACKEND="${RAG_BACKEND:-auto}"

log()  { printf '[rag-reindex] %s\n' "$*"; }
fail() { printf '[rag-reindex] FATAL: %s\n' "$*" >&2; exit "${2:-1}"; }

# ---------------------------------------------------------------------------
# Parse the PDO DSN into psql parameters. The DSN is the single source of truth
# (config/paths.php); deriving psql's arguments from it means the two can never
# drift into pointing at different databases.
# ---------------------------------------------------------------------------
DSN="${TRAKOPLENS_PG_DSN:-pgsql:host=127.0.0.1;port=5433;dbname=trakop_lens_vectors}"
PGH="$(printf '%s' "$DSN" | sed -n 's/.*host=\([^;]*\).*/\1/p')"
PGP="$(printf '%s' "$DSN" | sed -n 's/.*port=\([^;]*\).*/\1/p')"
PGDB="$(printf '%s' "$DSN" | sed -n 's/.*dbname=\([^;]*\).*/\1/p')"
PGU="${TRAKOPLENS_PG_USER:-trakop_lens}"
export PGPASSWORD="${TRAKOPLENS_PG_PASS:-}"
: "${PGH:=127.0.0.1}" "${PGP:=5433}" "${PGDB:=trakop_lens_vectors}"

psq() { psql -v ON_ERROR_STOP=1 -qtAX -h "$PGH" -p "$PGP" -U "$PGU" -d "$PGDB" "$@"; }

log "store: ${PGU}@${PGH}:${PGP}/${PGDB}"

[ -n "$PGPASSWORD" ] || fail "TRAKOPLENS_PG_PASS is empty. The app would fail OPEN
  (no error, no KB, no tool narrowing) — so this refuses instead of indexing into nothing."

# ---------------------------------------------------------------------------
# 1. Wait for the store. compose depends_on: service_healthy already gates this,
#    but `docker compose run` can be invoked directly against a starting stack.
# ---------------------------------------------------------------------------
for i in $(seq 1 60); do
    if pg_isready -h "$PGH" -p "$PGP" -U "$PGU" -d "$PGDB" -q 2>/dev/null; then break; fi
    [ "$i" = 1 ] && log "waiting for pgvector..."
    sleep 1
done
pg_isready -h "$PGH" -p "$PGP" -U "$PGU" -d "$PGDB" -q 2>/dev/null \
    || fail "pgvector unreachable at ${PGH}:${PGP} after 60s"

# ---------------------------------------------------------------------------
# 2. Converge the schema. /docker-entrypoint-initdb.d runs ONLY on an empty data
#    directory, so a volume that predates this compose file (for example the
#    hand-run standalone container) would never have it applied. Both files are
#    idempotent by design, so applying them on every reindex is safe and makes
#    "the schema is missing" impossible rather than merely documented.
# ---------------------------------------------------------------------------
for sql in "$APP_ROOT/db/pgvector/001_init.sql" "$APP_ROOT/db/pgvector/002_conversation_memory.sql"; do
    [ -f "$sql" ] || fail "schema file missing from the mount: $sql"
    log "applying $(basename "$sql") (idempotent)"
    psq -f "$sql" >/dev/null || fail "failed to apply $(basename "$sql")"
done

# ---------------------------------------------------------------------------
# 3. Dimension agreement. The vector column width and the embedding model's
#    output width are declared independently; a mismatch means every upsert
#    below would throw thousands of times.
# ---------------------------------------------------------------------------
EXPECT_DIM="${TRAKOPLENS_EMBED_DIM:-768}"
COL_DIM="$(psq -c "SELECT atttypmod FROM pg_attribute WHERE attrelid='lens_embeddings'::regclass AND attname='embedding'")"
[ "$COL_DIM" = "$EXPECT_DIM" ] \
    || fail "lens_embeddings.embedding is vector(${COL_DIM}) but TRAKOPLENS_EMBED_DIM=${EXPECT_DIM}.
  Changing the embedding model requires a FULL reindex into a matching column —
  not an incremental one, because lens_emb_uq is keyed on the model name and old
  rows would linger and never match." 1

# ---------------------------------------------------------------------------
# 3b. MySQL REACHABILITY. Two indexers read MySQL: SchemaIndexer (the table/column
#     descriptions) and QaIndexer (validated Q&A pairs grown from thumbs-up rated
#     answers). Both fail OPEN, so an unreachable database yields a quietly
#     PARTIAL index and this script would still exit 0 — `qa` is legitimately
#     allowed to be 0, so the row-count validation cannot catch it either.
#
#     This was not hypothetical: a reindex ran to completion against a container
#     that had no MySQL socket mounted, producing an index missing its Q&A corpus
#     with no error anywhere. Checked here, in the same way the app connects, so
#     the socket-vs-TCP distinction is exercised rather than assumed.
# ---------------------------------------------------------------------------
MYSQL_CHECK="$(cd "$APP_ROOT/webroot" && php -r '
$_SERVER["REQUEST_URI"]="/"; $_SERVER["HTTP_HOST"]="localhost";
$_SERVER["SERVER_ADDR"]="127.0.0.1"; $_SERVER["SERVER_NAME"]="localhost";
require "../config/paths.php";
try {
    $p = new PDO(sprintf("mysql:host=%s;dbname=%s", DBHOST, DBNAME), DBUSERNAME, DBPASSWORD,
         [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_TIMEOUT => 5]);
    $p->query("SELECT 1")->fetchColumn();
    printf("ok %s@%s", DBNAME, DBHOST);
} catch (Exception $e) {
    printf("FAIL %s@%s", DBNAME, DBHOST);
}' 2>/dev/null | tr -d '\r')"
case "$MYSQL_CHECK" in
    ok*) log "mysql: ${MYSQL_CHECK#ok }" ;;
    *)   fail "MySQL is unreachable (${MYSQL_CHECK#FAIL }).
  SchemaIndexer and QaIndexer read MySQL and both fail OPEN, so continuing would
  build a PARTIAL index and still report success. Note that config/paths.php uses
  DBHOST=\"localhost\", which means a UNIX SOCKET for PHP — the port is ignored —
  so this container needs the host's socket bind-mounted at
  /var/run/mysqld/mysqld.sock AND pdo_mysql.default_socket pointing at it." 1 ;;
esac

# ---------------------------------------------------------------------------
# 4. KB artefact. /docs/* never arrives through CI, so an unmounted snapshot is
#    the normal way the faq corpus silently ends up empty. Warn precisely here;
#    the row-count validation below is what actually fails the run.
# ---------------------------------------------------------------------------
SNAP="${TRAKOPLENS_KB_SNAPSHOT:-$APP_ROOT/docs/trakoplens-rag/kb-snapshot.dedup.ndjson}"
if [ -f "$SNAP" ]; then
    log "kb snapshot: $SNAP ($(wc -l < "$SNAP") lines), version=${TRAKOPLENS_KB_CORPUS_VERSION:-unversioned}"
    if [ "${TRAKOPLENS_KB_CORPUS_VERSION:-unversioned}" = "unversioned" ]; then
        log "NOTE: corpus version is 'unversioned'. Retrieval quality depends on which"
        log "      snapshot is loaded, so an unstamped corpus makes a QA result"
        log "      un-attributable. Set TRAKOPLENS_KB_CORPUS_VERSION when publishing one."
    fi
else
    log "WARNING: kb snapshot NOT found at $SNAP — the faq corpus will index only the in-code drafts."
fi

# ---------------------------------------------------------------------------
# 5. Run the indexers.
#    CWD MUST BE webroot. config/paths.php resolves the database from
#    getBetween(getcwd(), 'html/', 'webroot'), which only yields
#    'trakop-web/code/' from inside webroot. From code/ it yields
#    'trakop-web/code' (no trailing slash), nothing matches, and the SchemaIndexer
#    and QaIndexer would read the WRONG DATABASE — quietly indexing another
#    tenant's schema and Q&A pairs.
# ---------------------------------------------------------------------------
cd "$APP_ROOT/webroot" || fail "cannot enter $APP_ROOT/webroot"

# CakePHP's console SAPI has no HTTP_HOST — it's not a web request — so
# config/paths.php's per-domain db_records.txt lookup never matches, and
# SchemaIndexer/QaIndexer's live-discovery layer connects to the WRONG
# database (or none) and fails OPEN: schema_table quietly drops from ~158
# rows to the ~20 curated-only ones, with no error anywhere (observed for
# real — see ops/trakoplens-rag-deploy.md). The MySQL-reachability check
# above primes the same four values for its own throwaway PHP process; the
# actual reindex process needs them too. PHP's CLI SAPI populates $_SERVER
# from the process environment, so exporting them here is enough.
export HTTP_HOST=localhost REQUEST_URI=/ SERVER_ADDR=127.0.0.1 SERVER_NAME=localhost

# RETRIED, bounded. Embedding ~690 documents is a 20-30 minute job making one HTTP
# call per document, and EmbeddingClient has a fixed 15 s timeout with no retry of
# its own. A single slow response therefore aborts the entire run:
#
#     ... 100/135
#     Exception: embed_transport_failed: Operation timed out after 15001 milliseconds
#
# Observed for real when an unrelated embed request queued behind the reindex on a
# CPU-only Ollama: the model serialises requests, so ANY concurrent embedding work
# pushes latency past the client timeout and kills the job.
#
# Retrying here is safe because the store operation is an idempotent upsert keyed
# on (corpus, vendor_id, ref_key, model) — a repeated run rewrites the same rows
# rather than duplicating them. It is not a licence to hammer the embedder during
# a reindex (see ops/docker/DEPLOYMENT.md), but a transient blip 100 documents in
# should not fail a deploy step.
RAG_ATTEMPTS="${RAG_ATTEMPTS:-2}"
attempt=1
RC=1
while [ "$attempt" -le "$RAG_ATTEMPTS" ]; do
    log "running: bin/cake.php rag_reindex --corpus=${CORPUS} --backend=${BACKEND} (cwd=$(pwd), attempt ${attempt}/${RAG_ATTEMPTS})"
    set +e
    php ../bin/cake.php rag_reindex --corpus="$CORPUS" --backend="$BACKEND" 2>&1 \
        | grep -v '^\(PHP \)\?\(Notice\|Warning\|Deprecated\)' \
        | grep -v '^Deprecated Error:'
    RC="${PIPESTATUS[0]}"
    set -e
    [ "$RC" = "0" ] && break
    if [ "$attempt" -lt "$RAG_ATTEMPTS" ]; then
        log "attempt ${attempt} exited ${RC}; retrying — the upsert is idempotent, so rows already embedded are simply rewritten"
        sleep 5
    fi
    attempt=$((attempt + 1))
done
[ "$RC" = "0" ] || fail "rag_reindex exited ${RC} after ${RAG_ATTEMPTS} attempt(s)" 2

# ---------------------------------------------------------------------------
# 6. VALIDATE ROW COUNTS. The shell prints counts but always exits 0 — an
#    indexer that produced nothing (missing artefact, unreachable embedder,
#    an empty source directory) looks exactly like success. These minimums are
#    what turn that into a failed deploy step.
#
#    Defaults reflect what each corpus is: tool/schema/glossary/walkthrough are
#    built from the code and the tracked manifest, so zero always means broken.
#    `qa` is grown from thumbs-up rated answers on that server and is
#    legitimately zero on a fresh install, so its minimum is 0.
#    `faq` depends on the mounted artefact: its minimum is 1 by default (the
#    in-code drafts alone satisfy that) and should be raised per environment to
#    the real chunk count once a snapshot is published.
# ---------------------------------------------------------------------------
# The counts below are read from POSTGRES, so they only describe a run that built it.
# A MySQL-only build is validated by the shell's own exit code plus
# tools/trakoplens/verify-host-mode.sh; asserting pgvector row counts after it would
# either pass on a stale index or fail on a healthy one.
case "$BACKEND" in
    # A comma list ("pgvector,mysql-lexical") builds Postgres too, so match on substring
    # rather than on the three exact words — otherwise a perfectly good pgvector build
    # would skip its own row-count validation.
    *pgvector*|auto|all) ;;
    *)
        log ""
        log "backend=${BACKEND} did not build pgvector — skipping its row-count validation."
        log "verify that index with: php tools/trakoplens/host_status.php"
        log "corpus version: ${TRAKOPLENS_KB_CORPUS_VERSION:-unversioned}"
        exit 0 ;;
esac

MIN_TOOL="${RAG_MIN_TOOL:-1}"
MIN_FAQ="${RAG_MIN_FAQ:-1}"
MIN_GLOSSARY="${RAG_MIN_GLOSSARY:-1}"
MIN_WALKTHROUGH="${RAG_MIN_WALKTHROUGH:-1}"
MIN_SCHEMA="${RAG_MIN_SCHEMA_TABLE:-1}"
MIN_QA="${RAG_MIN_QA:-0}"

count_of() {
    psq -c "SELECT COALESCE(count(*),0) FROM lens_embeddings WHERE corpus = '$1'" 2>/dev/null || echo 0
}

log ""
log "corpus counts (minimums in brackets):"
FAILED=0
check() {
    local name="$1" min="$2" n
    n="$(count_of "$name")"
    n="${n:-0}"
    if [ "$n" -lt "$min" ]; then
        printf '[rag-reindex]   %-14s %-6s [min %s]  FAIL\n' "$name" "$n" "$min" >&2
        FAILED=1
    else
        printf '[rag-reindex]   %-14s %-6s [min %s]  ok\n' "$name" "$n" "$min"
    fi
}
check tool         "$MIN_TOOL"
check faq          "$MIN_FAQ"
check glossary     "$MIN_GLOSSARY"
check walkthrough  "$MIN_WALKTHROUGH"
check schema_table "$MIN_SCHEMA"
check qa           "$MIN_QA"

# Mixed-model rows are a distinct, silent failure: searches filter on the model
# name, so rows written under an older model are dead weight that will never
# match and are easy to mistake for a healthy corpus.
MODELS="$(psq -c "SELECT string_agg(DISTINCT model, ', ') FROM lens_embeddings")"
log ""
log "models present in the index: ${MODELS:-<none>}"
case "$MODELS" in
    *,*) log "WARNING: more than one embedding model is present. Searches filter on the model"
         log "         name, so rows from the other model can NEVER match. Truncate and reindex."
         ;;
esac

TOTAL="$(psq -c 'SELECT count(*) FROM lens_embeddings')"
log "total rows: ${TOTAL}   corpus version: ${TRAKOPLENS_KB_CORPUS_VERSION:-unversioned}"

if [ "$FAILED" = "1" ]; then
    fail "one or more corpora are below their minimum row count. The assistant will
  still answer (every retriever fails open) but without the missing corpus —
  which is why this exits non-zero instead of letting the deploy look clean." 3
fi

log "reindex complete and validated."
exit 0