#!/usr/bin/env bash # ============================================================================= # verify-stack.sh — post-deploy verification for the Trakop Docker stack. # # docker/scripts/verify-stack.sh # full run # docker/scripts/verify-stack.sh --no-llm # skip the live-answer check # docker/scripts/verify-stack.sh --no-reindex # skip re-running the ~25 min reindex # # NOTE: if your shell predates `usermod -aG docker`, run this under # sg docker -c 'docker/scripts/verify-stack.sh' # # WHY A SCRIPT AND NOT A CHECKLIST: # Nearly every failure this stack can have is SILENT. The retrievers fail # open, python errors are swallowed by `2>/dev/null`, and config/paths.php # falls through to a default database rather than erroring. A human running a # checklist sees a working login page in all of those states. Each check below # exists because the corresponding breakage produces no error anywhere. # # Exit 0 only if every non-skipped check passes. # ============================================================================= set -uo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." || exit 1 REPO_ROOT="$(pwd)" # --env-file is honoured if a .env.docker exists, matching DEPLOYMENT.md. ENV_FILE="" [ -f "$REPO_ROOT/.env.docker" ] && ENV_FILE="--env-file $REPO_ROOT/.env.docker" DC="docker compose $ENV_FILE" RUN_LLM=1 RUN_REINDEX=1 for arg in "$@"; do case "$arg" in --no-llm) RUN_LLM=0 ;; --no-reindex) RUN_REINDEX=0 ;; *) printf 'unknown argument: %s\n' "$arg" >&2; exit 2 ;; esac done PASS=0; FAIL=0; SKIP=0; WAIVE=0 G=$'\033[32m'; R=$'\033[31m'; Y=$'\033[33m'; N=$'\033[0m' ok() { printf " ${G}PASS${N} %-46s %s\n" "$1" "${2:-}"; PASS=$((PASS+1)); } skip() { printf " ${Y}SKIP${N} %-46s %s\n" "$1" "${2:-}"; SKIP=$((SKIP+1)); } # TRAKOP_VERIFY_WAIVE is a ';'-separated list of exact check names (the first # arg to bad()) to downgrade from FAIL to WAIVE — for a KNOWN environment # deviation only (e.g. a host that intentionally runs one service outside # Docker). Matched by exact string, no globbing, so a typo waives nothing. # Scope the CI variable to the staging environment only — never production; # the 172.16/12 address-pool check must NEVER be waived on any environment. is_waived() { case ";${TRAKOP_VERIFY_WAIVE:-};" in *";$1;"*) return 0;; esac; return 1; } bad() { if is_waived "$1"; then printf " ${Y}WAIVE${N} %-46s %s [TRAKOP_VERIFY_WAIVE]\n" "$1" "${2:-}" WAIVE=$((WAIVE+1)); return fi printf " ${R}FAIL${N} %-46s %s\n" "$1" "${2:-}"; FAIL=$((FAIL+1)); } # NOT named `head`: that would shadow the `head` COMMAND used in pipelines # below, silently turning `| head -1` into a call to this function. section() { printf '\n%s\n' "$1"; } # `systemctl is-enabled` prints the state AND exits non-zero for a disabled or # static unit, so `cmd || echo disabled` prints it TWICE. Only an ABSENT unit # produces empty output, so default on emptiness, never on exit status. enabled_state() { local s; s="$(systemctl is-enabled "$1" 2>/dev/null)"; printf '%s' "${s:-unknown}"; } active_state() { local s; s="$(systemctl is-active "$1" 2>/dev/null)"; printf '%s' "${s:-unknown}"; } # Read the values the stack is actually configured with. # # envval() strips INLINE COMMENTS and surrounding whitespace. A bare # `cut -d= -f2` does not, so `APACHE_PORT=8081 # host Apache keeps :80` produced # a "port" of `8081 # host Apache keeps :80` and three checks probed a # nonsense address while the stack was fine. Compose itself strips them, so the # file was valid and only this script was wrong — the worst kind of skew. envval() { grep -E "^$1=" .env.docker 2>/dev/null \ | tail -1 | cut -d= -f2- | sed 's/[[:space:]]#.*$//' \ | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' } PGPORT="$(envval PGVECTOR_PORT)"; : "${PGPORT:=5433}" OLPORT="$(envval OLLAMA_PORT)"; : "${OLPORT:=11434}" RDPORT="$(envval REDIS_PORT)"; : "${RDPORT:=6379}" APPORT="$(envval APACHE_PORT)"; : "${APPORT:=8081}" NDPORT="$(envval NODE_PORT)"; : "${NDPORT:=8400}" FLPORT="$(envval FLASK_PORT)"; : "${FLPORT:=5000}" EXPDIM="$(envval TRAKOPLENS_EMBED_DIM)"; : "${EXPDIM:=768}" EMODEL="$(envval OLLAMA_EMBED_MODEL)"; : "${EMODEL:=nomic-embed-text}" # The URL PREFIX the app answers on, which is NOT the same on every host, and # was hardcoded to the staging shape until 2026-09-04. # # staging .104 Apache DocumentRoot /var/www/html, app at /trakop-web/code/ # prod .144 the app MOVED TO THE DOMAIN ROOT on 2026-09-02 (see # URL-MOVE-2026-09-02.md). nginx 301s the old prefix and # Apache's DocumentRoot is itself .../trakop-web/code, so # requesting /trakop-web/code/ resolves to # .../code/trakop-web/code/ and is a genuine 404. # # Hardcoding one host's layout failed three checks against a perfectly healthy # prod stack — Apache was returning 200 with the right X-Trakop-Container all # along. A verifier that reports FAIL for a working system teaches people to # ignore it, which is worse than not checking. Default preserves .104 exactly. APPBASE="$(envval TRAKOP_APP_BASE)"; : "${APPBASE:=/trakop-web/code/}" case "$APPBASE" in */) ;; *) APPBASE="$APPBASE/" ;; esac printf '=== Trakop stack verification — %s ===\n' "$(date -u +%FT%TZ)" # --------------------------------------------------------------------------- section "Docker access" # --------------------------------------------------------------------------- if ! docker info >/dev/null 2>&1; then bad "docker daemon reachable" "permission denied or daemon down — add your user to the 'docker' group or use sudo" printf '\nCannot continue without docker access.\n' exit 1 fi ok "docker daemon reachable" # The 172.31 collision guard, checked at the DAEMON level. This is the single # most dangerous misconfiguration in this migration: Docker's default bridge # pool (172.17.0.0/16 .. 172.31.0.0/16) overlaps the production IPs hardcoded in # config/paths.php, and a container that draws one connects a dev box to # production MySQL with no error and no log line. pools="$(docker info --format '{{json .DefaultAddressPools}}' 2>/dev/null)" # OVERLAP, not the string "172.". The old test rejected any pool containing that # substring, which failed a staging server pinned to 172.20.0.0/14 — a pool that # spans 172.20-172.23 and therefore CANNOT hand out any of the 172.31.x production # addresses. A check that cries wolf on a safe configuration teaches operators to # ignore it, and this is the one check in this file that guards against silently # connecting a test box to the production database. # # The hazard is precise: Docker's DEFAULT pool is 172.17.0.0/16 .. 172.31.0.0/16, # which does contain them. So compute the actual intersection with the addresses # hardcoded in config/paths.php and nodeServer/server.js. PROD_IPS="172.31.12.223 172.31.33.124 172.31.43.197 172.31.15.66 172.31.36.244 172.31.43.14" if [ "$pools" = "null" ] || [ -z "$pools" ]; then bad "daemon address pool cannot reach the production IPs" \ "no explicit pool configured, so Docker uses its 172.17-172.31 default, which CONTAINS them — pin default-address-pools in /etc/docker/daemon.json (DEPLOYMENT.md step 0)" else clash="$(PROD_IPS="$PROD_IPS" POOLS="$pools" python3 - <<'PY' 2>/dev/null import ipaddress, json, os try: pools = json.loads(os.environ["POOLS"]) or [] except Exception: print("unparsable"); raise SystemExit hits = [] for p in pools: try: net = ipaddress.ip_network(p["Base"], strict=False) except Exception: continue for ip in os.environ["PROD_IPS"].split(): if ipaddress.ip_address(ip) in net: hits.append("%s in %s" % (ip, net)) print("; ".join(hits)) PY )" if [ "$clash" = "unparsable" ]; then skip "daemon address pool cannot reach the production IPs" "could not parse pools=$pools — check by hand" elif [ -n "$clash" ]; then bad "daemon address pool cannot reach the production IPs" \ "OVERLAP: $clash — a container drawing that address connects to PRODUCTION MySQL with no error" else ok "daemon address pool cannot reach the production IPs" "$pools" fi fi # --------------------------------------------------------------------------- section "1-2. Containers running" # --------------------------------------------------------------------------- for svc in cakephp node pgvector ollama redis adminer; do # TRAKOP_OLLAMA_HOST_MODE=1 on a host where Ollama runs as a host systemd # daemon (docker-compose.override.yml profiles the container out with # `!override`). This only skips the CONTAINER-presence assertion — the # "ollama API on :11434" / "model listed" / "embedding generation" checks # below probe 127.0.0.1 over HTTP regardless of container vs systemd, so # nothing is blinded. Mirrors the existing `mysql` host-daemon exemption. if [ "$svc" = "ollama" ] && [ "${TRAKOP_OLLAMA_HOST_MODE:-0}" = "1" ]; then skip "service up: ollama" "TRAKOP_OLLAMA_HOST_MODE=1 — host systemd daemon by design" continue fi state="$($DC ps --format '{{.Service}} {{.State}}' 2>/dev/null | awk -v s="$svc" '$1==s{print $2}')" if [ "$state" = "running" ]; then ok "service up: $svc"; else bad "service up: $svc" "state=${state:-absent}"; fi done # flask is `profiles: [utils]` — deliberately NOT part of `up -d`, because nothing in # the application references it. Absent is therefore CORRECT unless this server opted # in, so it is reported as a skip rather than a failure. FLASK_ENABLED=1 makes it # required on a server that does run it. FLASK_STATE="$($DC ps --format '{{.Service}} {{.State}}' 2>/dev/null | awk '$1=="flask"{print $2}')" if [ "$FLASK_STATE" = "running" ]; then ok "service up: flask" elif [ "${FLASK_ENABLED:-0}" = "1" ]; then bad "service up: flask" "FLASK_ENABLED=1 but state=${FLASK_STATE:-absent}" else skip "service up: flask" "opt-in (profile utils); not started on this server" fi # --------------------------------------------------------------------------- section "17. Health checks all pass" # --------------------------------------------------------------------------- # Checked before the individual probes: a healthcheck already encodes the # stricter "usable", and a container reporting healthy while a probe below fails # means the healthcheck itself is too weak. for svc in cakephp node flask pgvector ollama redis adminer; do cid="$($DC ps -q "$svc" 2>/dev/null | head -1)" if [ -z "$cid" ]; then skip "health: $svc" "not running"; continue; fi hs="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null)" case "$hs" in healthy) ok "health: $svc" ;; none) skip "health: $svc" "no healthcheck defined" ;; *) last="$(docker inspect --format '{{if .State.Health}}{{range .State.Health.Log}}{{.Output}}{{end}}{{end}}' "$cid" 2>/dev/null | tail -c 240 | tr '\n' ' ')" bad "health: $svc" "status=$hs $last" ;; esac done # --------------------------------------------------------------------------- section "3. CakePHP serves + Python executes + MySQL reachable from inside" # --------------------------------------------------------------------------- code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "http://127.0.0.1:${APPORT}${APPBASE}" 2>/dev/null)" # The root route redirects to login, so 2xx and 3xx are both correct; 5xx means # it answered with a fatal, 000 that nothing answered. case "$code" in 2??|3??) ok "cakephp HTTP on :$APPORT" "HTTP $code" ;; 000) bad "cakephp HTTP on :$APPORT" "no response" ;; *) bad "cakephp HTTP on :$APPORT" "HTTP $code" ;; esac # WHO answered? A 200 on :80 proves nothing on its own — host Apache and the # container serve the SAME source tree from the SAME path and are otherwise # indistinguishable over HTTP. Without this check the whole suite would pass # against host Apache and report a cutover that never happened. # # The container's vhost sets X-Trakop-Runtime: docker; host Apache's vhost lives # in /etc/apache2 on the host and is never touched by this migration, so it does # not. Presence is a positive identification. hdrs="$(curl -sI --max-time 10 "http://127.0.0.1:${APPORT}${APPBASE}" 2>/dev/null | tr -d '\r')" runtime="$(printf '%s' "$hdrs" | grep -i '^X-Trakop-Runtime:' | awk '{print $2}')" whichc="$(printf '%s' "$hdrs" | grep -i '^X-Trakop-Container:' | awk '{print $2}')" if [ "$runtime" = "docker" ]; then ok "port $APPORT is served by the CONTAINER" "X-Trakop-Container: ${whichc:-?}" elif [ -z "$runtime" ]; then bad "port $APPORT is served by the CONTAINER" "no X-Trakop-Runtime header — this is HOST Apache, not the container. Run: sudo docker/scripts/start-docker.sh" else bad "port $APPORT is served by the CONTAINER" "unexpected X-Trakop-Runtime: $runtime" fi # Python is asserted through the SAME mechanism the app uses — exec() of python3 # inside the PHP container — because that is where it breaks. The app's own call # ends in `2>/dev/null`, so a missing module surfaces only as `tool_failed`. if $DC exec -T cakephp python3 -c 'import pymysql,pandas,openpyxl,xlsxwriter,requests; print("ok")' 2>/dev/null | grep -q ok; then ok "python3 + delivery deps import" else bad "python3 + delivery deps import" "delivery tools shell out with 2>/dev/null, so this would appear only as tool_failed" fi # The delivery scripts are executed by PHP, not by a shell — verify PHP's exec() # path specifically, including that the script file is visible through the mount. pyexec="$($DC exec -T cakephp php -r ' $s = "/var/www/html/trakop-web/code/webroot/python_script/delivery_inventory.py"; if (!is_file($s)) { echo "missing_script"; exit; } exec("python3 -c \"import pymysql,pandas; print(42)\" 2>&1", $o, $rc); echo $rc === 0 && trim(implode("",$o)) === "42" ? "ok" : "rc={$rc}";' 2>/dev/null | tr -d '\r')" if [ "$pyexec" = "ok" ]; then ok "PHP exec() -> python3 works" else bad "PHP exec() -> python3 works" "$pyexec" fi # MySQL: the whole point of keeping it outside Docker is that the hop still # works. The container's own healthcheck asserts this too; here it is reported # with the resolved database name, which is what catches the getcwd()/IP traps. mysqlchk="$($DC exec -T cakephp php -r ' $_SERVER["REQUEST_URI"]="/"; $_SERVER["HTTP_HOST"]="localhost"; $_SERVER["SERVER_ADDR"]="127.0.0.1"; $_SERVER["SERVER_NAME"]="localhost"; chdir("/var/www/html/trakop-web/code/webroot"); require "/var/www/html/trakop-web/code/config/paths.php"; try { $p = new PDO(sprintf("mysql:host=%s;dbname=%s", DBHOST, DBNAME), DBUSERNAME, DBPASSWORD, [PDO::ATTR_ERRMODE=>PDO::ERRMODE_EXCEPTION, PDO::ATTR_TIMEOUT=>5]); $n = $p->query("SELECT COUNT(*) FROM master_configurations")->fetchColumn(); printf("ok db=%s host=%s vendors=%s", DBNAME, DBHOST, $n); } catch (Exception $e) { printf("FAIL db=%s host=%s", DBNAME, DBHOST); }' 2>/dev/null | tr -d '\r')" case "$mysqlchk" in ok*) ok "external MySQL reachable from container" "${mysqlchk#ok }" ;; *) bad "external MySQL reachable from container" "$mysqlchk" ;; esac # --------------------------------------------------------------------------- section "3b. Docker is the ONLY runtime (host web layer down, DB client containerised)" # --------------------------------------------------------------------------- # Two web servers and two PHP runtimes on one box is the drift this migration # exists to remove: a php.ini or vhost fixed in the container is silently # unfixed on the host, and whichever one holds :80 after a reboot decides which # of the two the users get. So "the host web layer is off" is a checked # property, not a one-time action. # # ENABLED is checked as well as ACTIVE. A stopped-but-enabled apache2 is a # time bomb: it reclaims :80 on the next reboot, before Docker starts. for u in apache2 php7.4-fpm; do if [ -z "$(systemctl list-unit-files --no-legend "$u.service" 2>/dev/null)" ]; then skip "host $u is off" "unit not installed" continue fi a="$(active_state "$u")" e="$(enabled_state "$u")" if [ "$a" != "active" ] && [ "$e" != "enabled" ]; then ok "host $u is off" "$a / $e" elif [ "$a" != "active" ]; then bad "host $u is off" "stopped but STILL ENABLED — it takes :80 back on the next reboot. Run: sudo systemctl disable $u" else bad "host $u is off" "still active ($a / $e) — run: sudo docker/scripts/start-docker.sh" fi done # MySQL is the deliberate exception. Asserted as PRESENT, because a "docker-only" # reading of this migration that stopped MySQL too would take the data with it. # LOCAL instance only. On a server whose database is on another host (staging reaches # it at 192.168.0.101) there is no mysql unit at all, and `inactive` is the correct # state — the reachability that matters is already asserted from INSIDE the container # above, with the resolved database name. Grading a remote-DB server on its local unit # reported a healthy stack as broken. mysql_unit="$(systemctl is-active mysql 2>/dev/null)" if [ "$mysql_unit" = "active" ]; then ok "host MySQL still running (external by design)" elif [ -z "$(systemctl list-unit-files --no-legend 'mysql*.service' 2>/dev/null)" ]; then skip "host MySQL still running" "no local mysql unit — this server's database is remote (see the reachability check above)" else bad "host MySQL still running (external by design)" "a local mysql unit exists but is ${mysql_unit:-unknown} — if the database is remote this check is not applicable" fi # The DB client. It replaced host Apache's /adminer.php, so if it is not usable # the cut-over removed the only way to inspect the database from a browser. ADMPORT="$(envval ADMINER_PORT)"; ADMPORT="${ADMPORT:-8082}" acode="$(curl -s -o /dev/null -w '%{http_code}' --max-time 8 "http://127.0.0.1:${ADMPORT}/" 2>/dev/null)" [ "$acode" = "200" ] && ok "adminer HTTP on 127.0.0.1:$ADMPORT" "HTTP 200" \ || bad "adminer HTTP on 127.0.0.1:$ADMPORT" "HTTP $acode" # Published on LOOPBACK only. The host adminer.php it replaces was reachable from # the whole LAN; a regression here re-exposes a database login form. if docker port trakop-adminer 2>/dev/null | grep -q '0\.0\.0\.0'; then bad "adminer is loopback-only" "published on 0.0.0.0 — a DB login form is exposed to the LAN" else ok "adminer is loopback-only" fi # The socket hop, which the login page renders perfectly without. if docker exec trakop-adminer bash /usr/local/bin/trakop-adminer-healthcheck >/dev/null 2>&1; then ok "adminer reaches MySQL over the mounted socket" else bad "adminer reaches MySQL over the mounted socket" "login page works but every login would fail — check mysqli.default_socket and the socket mount" fi # The storefront application, if this server has it. It moved INTO the cakephp # container when host Apache was disabled, and the failure mode is quiet: a # missing mount serves 404 from the very same URL that used to work. WEBDIR="$(envval WEBSITE_APP_DIR)" if [ -z "$WEBDIR" ] || case "$WEBDIR" in *no-website-app*) true ;; *) false ;; esac; then skip "storefront /cakephp/code/ served" "WEBSITE_APP_DIR not set on this server" else # Assert the tree is actually visible inside the container at the path its # own config/paths.php requires — `getBetween(getcwd(),"html/","webroot")` # must equal `cakephp/code/`, so the mount point is load-bearing, not cosmetic. if $DC exec -T cakephp test -f /var/www/html/cakephp/code/webroot/index.php 2>/dev/null; then ok "storefront tree mounted in the container" "/var/www/html/cakephp" else bad "storefront tree mounted in the container" "WEBSITE_APP_DIR=$WEBDIR did not land at /var/www/html/cakephp" fi scode="$(curl -s -o /dev/null -w '%{http_code}' --max-time 30 "http://127.0.0.1:${APPORT}/cakephp/code/" 2>/dev/null)" case "$scode" in # 302 to users/vendorNotFound is CORRECT for a request whose Host is not a # configured storefront domain — the vendor is resolved from HTTP_HOST via # this app's customerApiv3. 5xx means the framework itself failed to boot. 200|302) ok "storefront /cakephp/code/ answers" "HTTP $scode" ;; 404) bad "storefront /cakephp/code/ answers" "HTTP 404 — the mount is missing or empty" ;; *) bad "storefront /cakephp/code/ answers" "HTTP $scode" ;; esac fi # --------------------------------------------------------------------------- section "4-7. pgvector: reachable, extension, table populated, HNSW index" # --------------------------------------------------------------------------- psq() { $DC exec -T pgvector psql -qtAX -U "${PGVECTOR_USER:-trakop_lens}" -d "${PGVECTOR_DB:-trakop_lens_vectors}" -c "$1" 2>/dev/null | tr -d '\r'; } if [ -n "$(psq 'SELECT 1')" ]; then ok "pgvector reachable"; else bad "pgvector reachable" "psql failed inside the container"; fi ext="$(psq "SELECT extversion FROM pg_extension WHERE extname='vector'")" [ -n "$ext" ] && ok "pgvector extension present" "v$ext" || bad "pgvector extension present" "init SQL did not run" dim="$(psq "SELECT atttypmod FROM pg_attribute WHERE attrelid='lens_embeddings'::regclass AND attname='embedding'")" if [ "$dim" = "$EXPDIM" ]; then ok "vector width matches TRAKOPLENS_EMBED_DIM" "vector($dim)" else bad "vector width matches TRAKOPLENS_EMBED_DIM" "column=vector(${dim:-absent}) expected=$EXPDIM — a full reindex is required" fi rows="$(psq 'SELECT count(*) FROM lens_embeddings')" if [ "${rows:-0}" -gt 0 ] 2>/dev/null; then ok "lens_embeddings populated" "$rows rows" printf ' corpus breakdown: %s\n' "$(psq "SELECT string_agg(corpus||'='||n, ' ') FROM (SELECT corpus, count(*) n FROM lens_embeddings GROUP BY corpus ORDER BY corpus) t")" else bad "lens_embeddings populated" "0 rows — run: docker compose run --rm rag-reindex" fi hnsw="$(psq "SELECT count(*) FROM pg_indexes WHERE tablename='lens_embeddings' AND indexdef ILIKE '%USING hnsw%'")" if [ "${hnsw:-0}" -ge 1 ] 2>/dev/null; then ok "HNSW index exists" else bad "HNSW index exists" "present-but-unindexed changes retrieval RANKING as well as latency, and reports no error" fi # A second embedding model in the index is a distinct silent failure: searches # filter on the model name, so those rows can never match. models="$(psq "SELECT string_agg(DISTINCT model, ',') FROM lens_embeddings")" case "$models" in *,*) bad "single embedding model in index" "found: $models — rows from the other model can NEVER match; truncate and reindex" ;; "") skip "single embedding model in index" "index empty" ;; *) ok "single embedding model in index" "$models" ;; esac # --------------------------------------------------------------------------- section "8-9. Ollama reachable + model loaded" # --------------------------------------------------------------------------- tags="$(curl -fsS --max-time 5 "http://127.0.0.1:${OLPORT}/api/tags" 2>/dev/null)" if [ -n "$tags" ]; then ok "ollama API on :$OLPORT"; else bad "ollama API on :$OLPORT" "no response"; fi if printf '%s' "$tags" | grep -q "\"${EMODEL}"; then ok "model listed: $EMODEL"; else bad "model listed: $EMODEL" "not in /api/tags"; fi # 10. Embedding generation actually works at the right width. "Listed" and # "returns a 768-dim vector" are different claims and only the second is what # retrieval depends on — a model can be present and unloadable. # RETRIED ONCE. Ollama serialises embedding requests on CPU, so this check can # lose a race against any other embed in flight (a live question, a reindex) and # time out on a perfectly healthy model. A verifier that reports false failures is # worse than none — the next person debugs the wrong layer. One retry distinguishes # "busy" from "broken" without hiding a real fault: a genuinely dead model fails # both attempts. emb="" for _try in 1 2; do emb="$(curl -fsS --max-time 30 "http://127.0.0.1:${OLPORT}/api/embeddings" \ -H 'Content-Type: application/json' \ -d "{\"model\":\"${EMODEL}\",\"prompt\":\"search_query: verify\"}" 2>/dev/null)" printf '%s' "$emb" | grep -q '"embedding"' && break [ "$_try" = 1 ] && sleep 3 done embdim="$(printf '%s' "$emb" | sed -n 's/.*"embedding":\[\([^]]*\)\].*/\1/p' | tr ',' '\n' | grep -c '.' || true)" if [ "${embdim:-0}" = "$EXPDIM" ]; then ok "embedding generation works" "$embdim dims" else bad "embedding generation works" "got ${embdim:-0} dims, expected $EXPDIM" fi # --------------------------------------------------------------------------- section "11. Redis reachable" # --------------------------------------------------------------------------- if $DC exec -T redis redis-cli ping 2>/dev/null | grep -q PONG; then ok "redis PONG"; else bad "redis PONG" "no response"; fi # phpredis is what decides whether the Redis pool is real: without the extension # config/app.php silently falls back to the File engine — correct behaviour, but # then Redis is not actually in use and a "Redis is enabled" claim is false. engine="$($DC exec -T cakephp php -r ' $_SERVER["REQUEST_URI"]="/"; $_SERVER["HTTP_HOST"]="localhost"; $_SERVER["SERVER_ADDR"]="127.0.0.1"; $_SERVER["SERVER_NAME"]="localhost"; chdir("/var/www/html/trakop-web/code/webroot"); require "/var/www/html/trakop-web/code/config/paths.php"; require "/var/www/html/trakop-web/code/vendor/autoload.php"; require "/var/www/html/trakop-web/code/vendor/cakephp/cakephp/src/Core/functions.php"; $c = require "/var/www/html/trakop-web/code/config/app.php"; printf("%s|%s|%s", extension_loaded("redis")?"ext":"noext", TRAKOPLENS_CACHE_ENGINE, $c["Cache"]["trakoplens"]["className"]); ' 2>/dev/null | tr -d '\r')" case "$engine" in ext\|redis\|Redis) ok "cache pool 'trakoplens' on Redis" "$engine" ;; noext\|redis*) bad "cache pool 'trakoplens' on Redis" "phpredis MISSING; pool fell back to File. Rebuild the image." ;; *\|file\|File) skip "cache pool 'trakoplens' on Redis" "TRAKOPLENS_CACHE_ENGINE=file (deliberate)" ;; *) bad "cache pool 'trakoplens' on Redis" "unexpected: $engine" ;; esac # A write/read round trip through Cake's own Cache layer — the config being # right and the pool working are separate facts. if $DC exec -T cakephp php -r ' $_SERVER["REQUEST_URI"]="/"; $_SERVER["HTTP_HOST"]="localhost"; $_SERVER["SERVER_ADDR"]="127.0.0.1"; $_SERVER["SERVER_NAME"]="localhost"; chdir("/var/www/html/trakop-web/code/webroot"); require "/var/www/html/trakop-web/code/vendor/autoload.php"; require "/var/www/html/trakop-web/code/config/bootstrap.php"; $k = "verify_" . getmypid(); Cake\Cache\Cache::write($k, "roundtrip", "trakoplens"); echo Cake\Cache\Cache::read($k, "trakoplens") === "roundtrip" ? "ok" : "mismatch"; Cake\Cache\Cache::delete($k, "trakoplens");' 2>/dev/null | grep -q ok; then ok "cache write/read round trip" else bad "cache write/read round trip" "Cache::write/read through the 'trakoplens' pool failed" fi # --------------------------------------------------------------------------- section "12. Socket.IO" # --------------------------------------------------------------------------- hs="$(curl -fsS --max-time 5 "http://127.0.0.1:${NDPORT}/socket.io/?EIO=4&transport=polling" 2>/dev/null)" if printf '%s' "$hs" | grep -q '"sid"'; then ok "socket.io handshake on :$NDPORT"; else bad "socket.io handshake on :$NDPORT" "no sid in the response"; fi # Express + its per-request Sequelize connection. Done ONCE here rather than in # the recurring healthcheck, because the global middleware opens and closes a # MySQL connection on every request to `/`. root="$(curl -fsS --max-time 10 "http://127.0.0.1:${NDPORT}/" 2>/dev/null)" if printf '%s' "$root" | grep -q 'Welcome to trakop'; then ok "node express + per-request DB middleware"; else bad "node express + per-request DB middleware" "unexpected body from /"; fi # --------------------------------------------------------------------------- section "12b. Flask utility app" # --------------------------------------------------------------------------- if [ "$FLASK_STATE" != "running" ]; then # Something may still answer that port (another site, a host-native Flask). Report # that fact instead of grading OUR container on a stranger's response. other="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "http://127.0.0.1:${FLPORT}/" 2>/dev/null)" if [ "$other" != "000" ] && [ -n "$other" ]; then skip "flask serving on :$FLPORT" "flask container not started; something ELSE answers :$FLPORT (HTTP $other) — not this stack" else skip "flask serving on :$FLPORT" "flask container not started (opt-in profile utils)" fi else fl="$(curl -fsS --max-time 10 "http://127.0.0.1:${FLPORT}/" 2>/dev/null)" if printf '%s' "$fl" | grep -q 'Welcome to home'; then ok "flask serving on :$FLPORT"; else bad "flask serving on :$FLPORT" "unexpected body"; fi fi # gunicorn, not the Werkzeug dev server. run.py's `app.run(debug=True)` enables an # interactive debugger that executes arbitrary Python from the browser on any # traceback — it must never be what is listening. srv="" [ "$FLASK_STATE" = "running" ] && srv="$(curl -sI --max-time 5 "http://127.0.0.1:${FLPORT}/" 2>/dev/null | grep -i '^server:' | tr -d '\r')" case "$srv" in *gunicorn*) ok "flask served by gunicorn" "$srv" ;; *Werkzeug*) bad "flask served by gunicorn" "WERKZEUG DEV SERVER IS LISTENING — debug=True is a remote code execution hole" ;; *) skip "flask served by gunicorn" "no Server header: ${srv:-none}" ;; esac # --------------------------------------------------------------------------- section "13-15. Retrieval, reindex, and a real grounded answer" # --------------------------------------------------------------------------- # Retrieval through the APPLICATION's own Retriever, not a raw SQL similarity # query. tool_hits=0 with RAG_MODE=active is the exact signature of a silent # fail-open, and it is invisible on every other surface. # # `tools=0` on its own does not say WHY, and the causes need opposite responses: # a dead embedder, an unreachable Postgres, a model/dim mismatch and a corpus # that was never indexed all present identically. The probe therefore also # reports Retriever::lastError() — the throwable the fail-open swallowed, empty # when the call completed normally — and the indexed tool-row count, so the FAIL # line names the actual fault instead of restating the symptom. rag_probe() { $DC exec -T cakephp php -r ' $_SERVER["REQUEST_URI"]="/"; $_SERVER["HTTP_HOST"]="localhost"; $_SERVER["SERVER_ADDR"]="127.0.0.1"; $_SERVER["SERVER_NAME"]="localhost"; chdir("/var/www/html/trakop-web/code/webroot"); require "/var/www/html/trakop-web/code/vendor/autoload.php"; require "/var/www/html/trakop-web/code/config/bootstrap.php"; $t = (new App\Lib\TrakopLens\Rag\Retriever())->retrieveTools("how many orders today", 0, 8); $indexed = "?"; try { $indexed = "0"; foreach ((new App\Lib\TrakopLens\Rag\VectorStore())->countByCorpus() as $row) { if ($row["corpus"] === "tool") { $indexed = (string)$row["n"]; } } } catch (\Throwable $e) { } printf("mode=%s tools=%d indexed=%s err=%s", TRAKOPLENS_RAG_MODE, count($t), $indexed, str_replace("\n", " ", (string)App\Lib\TrakopLens\Rag\Retriever::lastError()));' 2>/dev/null | tr -d '\r' | tr '\n' ' ' } retr="$(rag_probe)" tools_n="$(printf '%s' "$retr" | sed -n 's/.*tools=\([0-9]*\).*/\1/p')" # A transient embed timeout on a loaded host fails open in exactly the same way # as a broken config. Retry ONCE before grading, so a momentary latency spike is # not reported as a stack fault — and so anything still failing is known to # reproduce. The first-attempt failure is still printed: silently absorbing it # would hide a host that is one spike away from dropping RAG in production. if [ "${tools_n:-0}" -eq 0 ] 2>/dev/null && ! printf '%s' "$retr" | grep -q 'mode=off'; then sleep 3 retr2="$(rag_probe)" tools_n2="$(printf '%s' "$retr2" | sed -n 's/.*tools=\([0-9]*\).*/\1/p')" if [ "${tools_n2:-0}" -gt 0 ] 2>/dev/null; then printf ' first attempt failed open, retry succeeded — %s\n' "$retr" fi retr="$retr2"; tools_n="$tools_n2" fi rag_err="$(printf '%s' "$retr" | sed -n 's/.*err=//p' | sed 's/[[:space:]]*$//')" rag_indexed="$(printf '%s' "$retr" | sed -n 's/.*indexed=\([0-9?]*\).*/\1/p')" if [ "${tools_n:-0}" -gt 0 ] 2>/dev/null; then ok "tool retrieval returns hits" "$retr" elif printf '%s' "$retr" | grep -q 'mode=off'; then skip "tool retrieval returns hits" "RAG_MODE=off (deliberate)" elif [ -z "$retr" ]; then bad "tool retrieval returns hits" "probe produced NO output — the cakephp exec itself failed (missing --env-file/.env.docker?)" elif [ -n "$rag_err" ]; then bad "tool retrieval returns hits" "retrieval threw and failed open — $rag_err" elif [ "${rag_indexed:-0}" = "0" ]; then bad "tool retrieval returns hits" "tool corpus is EMPTY (indexed=0) — run the rag-reindex" else bad "tool retrieval returns hits" "$rag_indexed tools indexed, nothing thrown, yet 0 hits — check the embedding model/dim matches the index" fi # The reindex is verified by RUNNING it: it is idempotent (upsert keyed on # corpus+vendor+ref_key+model), and its own exit code already encodes the # per-corpus row-count validation. if [ "$RUN_REINDEX" = "0" ]; then skip "rag-reindex succeeds and validates" "--no-reindex" : > /tmp/trakop-reindex.$$ elif true; then printf ' running: docker compose run --rm rag-reindex (idempotent, ~25 min)\n' printf ' NOTE: Ollama serialises embeds on CPU. Do not issue other embedding\n' printf ' requests during this step — the reindex runs with a raised\n' printf ' TRAKOPLENS_EMBED_TIMEOUT_MS, but contention still slows it.\n' if $DC --profile tools run --rm rag-reindex >/tmp/trakop-reindex.$$ 2>&1; then ok "rag-reindex succeeds and validates" "$(grep -c 'ok$' /tmp/trakop-reindex.$$ 2>/dev/null || echo '?') corpora within minimums" else rc=$? bad "rag-reindex succeeds and validates" "exit $rc — see /tmp/trakop-reindex.$$" fi fi grep -E '^\[rag-reindex\] (tool|faq|glossary|walkthrough|schema_table|qa)' /tmp/trakop-reindex.$$ 2>/dev/null | sed 's/^/ /' # A real end-to-end answer needs the external LLM gateway, which is neither # containerised nor free to call. Opt out with --no-llm in CI. if [ "$RUN_LLM" = "1" ]; then printf ' NOTE: the answer check calls the EXTERNAL LLM gateway (optimize.trakop.com).\n' if [ -f "$REPO_ROOT/code/tests/TrakopLens/run_checks.php" ]; then if $DC exec -T -w /var/www/html/trakop-web/code/webroot cakephp php ../tests/TrakopLens/run_checks.php >/tmp/trakop-checks.$$ 2>&1; then ok "TrakopLens answer checks" "$(tail -3 /tmp/trakop-checks.$$ | tr '\n' ' ')" else # This harness has pre-existing unrelated failures on this branch, so # a non-zero exit is reported as a warning to inspect, not a stack # verdict — the stack-level facts are the checks above. skip "TrakopLens answer checks" "harness exited non-zero (it has known pre-existing failures) — inspect /tmp/trakop-checks.$$" fi else skip "TrakopLens answer checks" "tests/TrakopLens/run_checks.php not present" fi else skip "TrakopLens answer checks" "--no-llm" fi # --------------------------------------------------------------------------- section "16. Kong inbound gateway is intact" # --------------------------------------------------------------------------- # These two checks used to assert the OPPOSITE — that the gateway had been deleted. # That was reversed on review: the inbound gateway stays, so the checks now guard # against it being dropped again. GatewayApiController and TRAKOPLENS_KONG_KEY are a # PAIR: the controller authorises every request against the constant, so a tree with # one and not the other fatals on "undefined constant" instead of returning 403. # Both must be present, or both absent — never one. if [ -f "$REPO_ROOT/code/src/Controller/GatewayApiController.php" ]; then ok "GatewayApiController present" else bad "GatewayApiController present" "file missing — the inbound gateway was removed" fi if grep -rqE "define\s*\(\s*['\"]TRAKOPLENS_KONG_KEY" "$REPO_ROOT/code/config" 2>/dev/null; then ok "TRAKOPLENS_KONG_KEY defined (the controller cannot authorise without it)" else bad "TRAKOPLENS_KONG_KEY defined (the controller cannot authorise without it)" "not defined in code/config" fi # Requirement: the gateway ADDRESS comes from TRAKOPLENS_KONG_BASE, and the configured # port must be detectable. Resolved the same way the constant resolves at runtime — an # exported env var wins over the fallback literal in paths.php — so this cannot report a # port the app would not actually use. A missing/unparseable constant is a FAILURE (the # address is part of the kept configuration), not a silent default back to :8000. KONGDEF="$(grep -E "define\s*\(\s*'TRAKOPLENS_KONG_BASE'" "$REPO_ROOT/code/config/paths.php" 2>/dev/null | head -1)" KONGBASE="${TRAKOPLENS_KONG_BASE:-$(printf '%s' "$KONGDEF" | sed -nE "s/.*\?:[[:space:]]*'([^']+)'.*/\\1/p")}" KONGPORT="$(printf '%s' "$KONGBASE" | sed -nE 's#^[a-z]+://[^/:]+:([0-9]+)/?.*$#\1#p')" if [ -z "$KONGBASE" ]; then bad "TRAKOPLENS_KONG_BASE resolves" "constant missing or unparseable in code/config/paths.php" elif [ -z "$KONGPORT" ]; then ok "TRAKOPLENS_KONG_BASE resolves" "$KONGBASE (scheme default port)" else ok "TRAKOPLENS_KONG_BASE resolves" "$KONGBASE (port $KONGPORT)" fi # A listener on the gateway's own port is EXPECTED, not a defect. This check used to FAIL # when :8000 answered, which was correct only while the gateway was being deleted; with the # gateway kept, a running Kong would have failed the whole run. Kong B is NOT required to # be serving for the structural checks above to pass, so absence is SKIP, never FAIL — the # chat widget does not use this path and nothing degrades when no proxy is up. if [ -n "$KONGPORT" ]; then if (exec 3<>/dev/tcp/127.0.0.1/"$KONGPORT") 2>/dev/null; then ok "inbound gateway proxy on :$KONGPORT" "listener present" else skip "inbound gateway proxy on :$KONGPORT" "no proxy running — /gateway-api/* still serves directly" fi else skip "inbound gateway proxy" "no explicit port in $KONGBASE" fi # The two checks above prove the CODE and the CONFIG are present; this proves the endpoint # is still ROUTED (DashedRoute fallback -> GatewayApiController) and executing. 401/403 is # the HEALTHY answer: the key gate ran and refused. 404 means the controller stopped being # reachable and 5xx means it fatals -- neither of which a file-exists grep can see. Kong B # does not have to be up for this: it hits the app directly. gcode="$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 \ "http://127.0.0.1:${APPORT}${APPBASE}gateway-api/health-check?vendor_id=1" 2>/dev/null)" case "$gcode" in 401|403) ok "/gateway-api/* routed and fail-closed" "HTTP $gcode from the key gate" ;; 200) ok "/gateway-api/* routed and authorised" "HTTP 200 — a real key is configured" ;; # Not a gateway fault: the app itself is not answering, which section 3 already failed on. 000) skip "/gateway-api/* routed" "app not answering on :$APPORT" ;; *) bad "/gateway-api/* routed and fail-closed" "HTTP $gcode — expected 401/403 from _gatewayKeyError()" ;; esac # --------------------------------------------------------------------------- printf '\n=====================================================\n' printf ' PASS %s FAIL %s SKIP %s WAIVED %s\n' "$PASS" "$FAIL" "$SKIP" "$WAIVE" printf '=====================================================\n' [ "$WAIVE" -eq 0 ] || printf ' WAIVED CHECKS WERE NOT VERIFIED: %s\n' "${TRAKOP_VERIFY_WAIVE:-}" [ "$FAIL" -eq 0 ] || exit 1