feat(AI): add opt-in relevance check for off-topic retrieval

The post-rerank relevance floor rejects noise but passes coherent
off-topic questions, and it gets worse as the library grows. Adding
Gutenberg prose, the Python tutorial and MDN pages to the eval corpus
took the off-topic leak rate from 22% to 67% at the 0.62 floor.

Score cutoff does not fix it. The weakest correct top hit scores 0.666
while "write me a haiku about autumn leaves" pulls Walden at 0.705.
A cosine floor and a top-vs-median contrast test both cost recall
faster than they cut leaks.

- Add rag.relevanceCheck (default off). After the floor, the tasks
  model, or the chat model when none is set, gives one grammar-constrained
  on_topic verdict for the whole retrieved set. Off topic means no
  context and no citations. Any failure keeps the chunks unjudged.
- Add a Settings > Models toggle recommending an 8B+ tasks model.
  On llama3:8b the check cuts off-topic leaks from 67% to 6% for about
  5 points of core recall@5 and ~0.6s per turn. On 3B models it rejects
  most valid questions, which is why it is opt-in.
- Add 6 distractor documents and an off_topic suite (18 unanswerable
  questions, 7 distractor controls) to the eval corpus.
- Report per-tag leak rates and leaked cases with top scores in
  eval:retrieval. Add --judge-model to measure the check and --dump to
  export pre-floor candidate pools for offline gate simulation.
- Pin the check off in eval:generation so a local setting cannot move
  scores.
- Record the distractor sweep on RAG_MIN_FINAL_SCORE. The 0.62 default
  stays: 0.65 halves leaks at no measured recall cost but leaves only
  0.016 of margin.

The corpus fingerprint changes to b4ebe5dce8699f5d. Retrieval reports
from before this commit are not comparable, and a new baseline is
committed.

Refs #1341

admin/app/controllers/settings_controller.ts

@@ -9,6 +9,7 @@ import { inject } from '@adonisjs/core'
 import type { HttpContext } from '@adonisjs/core/http'
 import env from '#start/env'
 import { parseMinRelevance } from '../utils/misc.js'
+import { isRelevanceCheckEnabled } from '../utils/relevance_judge.js'
 import { parseResponseStyle } from '../utils/sampler.js'
 import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js'
 
@@ -80,6 +81,7 @@ export default class SettingsController {
     const ragEnabled = await KVStore.getValue('rag.enabled')
     const contextWindow = await KVStore.getValue('ai.contextWindow')
     const minRelevance = await KVStore.getValue('rag.minRelevance')
+    const relevanceCheck = await KVStore.getValue('rag.relevanceCheck')
     const responseStyle = await KVStore.getValue('ai.responseStyle')
     // Resolved window per installed model, so the setting shows what "Auto"
     // actually produced rather than leaving the user to guess. Best-effort:
@@ -110,6 +112,7 @@ export default class SettingsController {
           // Sent as the resolved number so the select can match an option
           // without duplicating the "unset means the default" rule in the UI.
           minRelevance: parseMinRelevance(minRelevance, RAG_MIN_FINAL_SCORE),
+          relevanceCheck: isRelevanceCheckEnabled(relevanceCheck),
           // Resolved rather than raw, for the same reason: unset means 'auto',
           // and the select shouldn't have to know that.
           responseStyle: parseResponseStyle(responseStyle),

admin/app/services/eval_generation_service.ts

@@ -296,6 +296,9 @@ export class EvalGenerationService {
       skipQueryRewrite: options.skipQueryRewrite || isMock,
       // A developer's personal NOMAD.md would silently skew every score.
       skipNomadMd: true,
+      // Pinned off rather than read from `rag.relevanceCheck`, for the same
+      // reason as minFinalScore. eval:retrieval --judge-model measures it.
+      relevanceCheckModel: null,
     }
 
     if (options.mode === 'oracle') {

admin/app/services/eval_retrieval_service.ts

@@ -1,5 +1,6 @@
 import { EvalCorpusService } from '#services/eval_corpus_service'
 import { RagService } from '#services/rag_service'
+import { RelevanceJudgeService } from '#services/relevance_judge_service'
 import { inject } from '@adonisjs/core'
 import { KB_EVAL_COLLECTION } from '../../constants/kb_collections.js'
 import {
@@ -29,6 +30,33 @@ export type RetrievalRunOptions = {
   kValues?: number[]
   /** Score the raw dense / reranked / diversified orderings separately. */
   ablate?: boolean
+  /** Also return every golden's full pre-floor candidate pool (see RetrievalStages.candidates). */
+  dump?: boolean
+  /**
+   * Run the relevance check on this model after the floor, as chat does when it
+   * is enabled. Unset keeps this tier model-free and deterministic.
+   */
+  judgeModel?: string
+}
+
+/** What the relevance check cost and did, when `judgeModel` was set. */
+export type JudgeStats = {
+  model: string
+  calls: number
+  /** Calls that failed open (timeout, error, unusable verdict). */
+  failures: number
+  rejected: number
+  meanMs: number
+  maxMs: number
+}
+
+/** One golden's full candidate pool, for simulating a relevance gate offline. */
+export type CandidatePool = {
+  id: string
+  tags: string[]
+  expectRefusal: boolean
+  relevantDocIds: string[]
+  candidates: Array<{ docId: string | null; score: number; semanticScore: number | null }>
 }
 
 export type StageAblation = {
@@ -48,6 +76,9 @@ export type RetrievalRunResult = {
    * collection filter leaked and the run is measuring the wrong corpus.
    */
   unresolvedChunks: number
+  /** Every golden's candidate pool, when `dump` was requested. */
+  pools: CandidatePool[] | null
+  judge: JudgeStats | null
 }
 
 /**
@@ -69,7 +100,8 @@ export type RetrievalRunResult = {
 export class EvalRetrievalService {
   constructor(
     private ragService: RagService,
-    private corpusService: EvalCorpusService
+    private corpusService: EvalCorpusService,
+    private judgeService: RelevanceJudgeService
   ) {}
 
   async run(goldens: Golden[], options: RetrievalRunOptions = {}): Promise<RetrievalRunResult> {
@@ -87,19 +119,32 @@ export class EvalRetrievalService {
     const denseCases: RetrievalCase[] = []
     const rerankedCases: RetrievalCase[] = []
     const diversifiedCases: RetrievalCase[] = []
+    const pools: CandidatePool[] = []
+    const recordStages = options.ablate || options.dump
+    const judgeMs: number[] = []
+    let judgeFailures = 0
+    let judgeRejected = 0
     let unresolvedChunks = 0
 
     for (const golden of goldens) {
       const stages: RetrievalStages = {}
-      const docs = await this.ragService.searchSimilarDocuments(
+      let docs = await this.ragService.searchSimilarDocuments(
         golden.query,
         topK,
         scoreThreshold,
         KB_EVAL_COLLECTION,
-        options.ablate ? stages : undefined,
+        recordStages ? stages : undefined,
         minFinalScore
       )
 
+      if (options.judgeModel && docs.length > 0) {
+        const verdict = await this.judgeService.judge(golden.query, docs, options.judgeModel)
+        judgeMs.push(verdict.ms)
+        if (!verdict.judged) judgeFailures++
+        judgeRejected += verdict.rejected
+        docs = verdict.kept
+      }
+
       const retrieved: ScoredChunk[] = docs.map((d) => {
         const docId = docIdFromSource(d.metadata?.source)
         if (!docId) unresolvedChunks++
@@ -108,6 +153,20 @@ export class EvalRetrievalService {
 
       cases.push(toCase(golden, retrieved))
 
+      if (options.dump) {
+        pools.push({
+          id: golden.id,
+          tags: golden.tags,
+          expectRefusal: golden.expectRefusal,
+          relevantDocIds: golden.relevantDocIds,
+          candidates: (stages.candidates ?? []).map((c) => ({
+            docId: docIdFromSource(c.source),
+            score: c.score,
+            semanticScore: c.semanticScore ?? null,
+          })),
+        })
+      }
+
       if (options.ablate) {
         denseCases.push(toCase(golden, stageToChunks(stages.dense)))
         rerankedCases.push(toCase(golden, stageToChunks(stages.reranked)))
@@ -133,6 +192,17 @@ export class EvalRetrievalService {
           }
         : null,
       unresolvedChunks,
+      pools: options.dump ? pools : null,
+      judge: options.judgeModel
+        ? {
+            model: options.judgeModel,
+            calls: judgeMs.length,
+            failures: judgeFailures,
+            rejected: judgeRejected,
+            meanMs: judgeMs.length ? judgeMs.reduce((a, b) => a + b, 0) / judgeMs.length : 0,
+            maxMs: judgeMs.length ? Math.max(...judgeMs) : 0,
+          }
+        : null,
     }
   }
 

admin/app/services/rag_pipeline_service.ts

@@ -2,6 +2,7 @@ import { ContextWindowService } from '#services/context_window_service'
 import { NomadMdService } from '#services/nomad_md_service'
 import { OllamaService } from '#services/ollama_service'
 import { RagService } from '#services/rag_service'
+import { RelevanceJudgeService } from '#services/relevance_judge_service'
 import { TokenCalibrationService } from '#services/token_calibration_service'
 import { inject } from '@adonisjs/core'
 import logger from '@adonisjs/core/services/logger'
@@ -43,7 +44,8 @@ export class RagPipelineService {
     private ragService: RagService,
     private nomadMdService: NomadMdService,
     private contextWindowService: ContextWindowService,
-    private tokenCalibration: TokenCalibrationService
+    private tokenCalibration: TokenCalibrationService,
+    private relevanceJudge: RelevanceJudgeService
   ) {}
 
   /**
@@ -104,9 +106,11 @@ export class RagPipelineService {
       numCtx: undefined,
       numPredict: undefined,
       contextLimits: { maxResults: RAG_DEFAULT_TOP_K, maxTokens: 0 },
-      timings: { rewriteMs: 0, retrievalMs: 0 },
+      timings: { rewriteMs: 0, retrievalMs: 0, relevanceCheckMs: 0 },
       minFinalScore: 0,
       chunksBelowFloor: 0,
+      relevanceCheckModel: null,
+      chunksRejectedByCheck: 0,
     }
 
     // --- Retrieval -------------------------------------------------------
@@ -157,6 +161,25 @@ export class RagPipelineService {
               ? ` (${floor.belowFloor} of ${floor.candidates} dropped below the ${minFinalScore} relevance floor)`
               : '')
         )
+
+        // Opt-in relevance check (#1341). The floor cannot tell a coherent
+        // off-topic question from a real one on a large general-prose library;
+        // a model can, at ~8B. Judged against the rewritten query, which is the
+        // standalone form of what the user is asking this turn.
+        if (relevantDocs.length > 0) {
+          const checkModel =
+            opts.relevanceCheckModel !== undefined
+              ? opts.relevanceCheckModel
+              : await this.relevanceJudge.resolveModel(model)
+          if (checkModel) {
+            const verdict = await this.relevanceJudge.judge(retrievalQuery, relevantDocs, checkModel)
+            relevantDocs = verdict.kept
+            trace.retrieved = relevantDocs
+            trace.relevanceCheckModel = checkModel
+            trace.chunksRejectedByCheck = verdict.rejected
+            trace.timings.relevanceCheckMs = verdict.ms
+          }
+        }
       }
     }
 

admin/app/services/rag_service.ts

@@ -1112,6 +1112,11 @@ export class RagService {
         stagesOut.diversified = diverseResults
           .slice(0, limit)
           .map((r) => ({ source: r.source, score: r.finalScore }))
+        stagesOut.candidates = rerankedResults.map((r) => ({
+          source: r.source,
+          score: r.finalScore,
+          semanticScore: r.score,
+        }))
       }
 
       // Return top N results with enhanced metadata

admin/app/services/relevance_judge_service.ts

@@ -0,0 +1,121 @@
+import { OllamaService } from '#services/ollama_service'
+import { inject } from '@adonisjs/core'
+import logger from '@adonisjs/core/services/logger'
+import KVStore from '#models/kv_store'
+import { RELEVANCE_CHECK_MAX_TOKENS, SYSTEM_PROMPTS } from '../../constants/ollama.js'
+import type { RetrievedChunk } from '../../types/rag.js'
+import {
+  buildJudgeInput,
+  isRelevanceCheckEnabled,
+  ON_TOPIC_SCHEMA,
+  pickOnTopic,
+} from '../utils/relevance_judge.js'
+import { resolveStructured } from '../utils/structured_output.js'
+import { resolveTasksModel } from '../utils/tasks_model.js'
+
+export type JudgeResult = {
+  /** Every chunk when the model said they fit the question, none when it said not. */
+  kept: RetrievedChunk[]
+  /** How many were dropped. 0 when the check failed open. */
+  rejected: number
+  /** False when the call failed and every chunk was kept unjudged. */
+  judged: boolean
+  ms: number
+}
+
+/**
+ * Asks a model whether the retrieved chunks are about what the user asked, and
+ * drops them all when it says no (#1341). Runs after the relevance floor, so it
+ * only ever sees the few chunks that were about to be injected and cited.
+ *
+ * Fails open by design. The floor has already removed the noise; this removes
+ * the coherent-but-off-topic remainder. A timeout or a garbled verdict costing
+ * one turn that precision is a much smaller failure than an answerable question
+ * losing its context because a model misbehaved.
+ */
+@inject()
+export class RelevanceJudgeService {
+  constructor(private ollamaService: OllamaService) {}
+
+  /**
+   * The model to run the check on this turn, or null to skip it. Opt-in via
+   * `rag.relevanceCheck`; runs on the tasks model when one is configured and on
+   * the chat model otherwise.
+   */
+  async resolveModel(chatModel: string): Promise<string | null> {
+    try {
+      if (!isRelevanceCheckEnabled(await KVStore.getValue('rag.relevanceCheck'))) return null
+    } catch (error) {
+      logger.warn(
+        `[RAG] Failed to read rag.relevanceCheck, skipping the check: ${error instanceof Error ? error.message : error}`
+      )
+      return null
+    }
+    return resolveTasksModel(this.ollamaService, chatModel, undefined, '[RAG]')
+  }
+
+  async judge(question: string, chunks: RetrievedChunk[], model: string): Promise<JudgeResult> {
+    const started = Date.now()
+    const unjudged = (): JudgeResult => ({
+      kept: chunks,
+      rejected: 0,
+      judged: false,
+      ms: Date.now() - started,
+    })
+    if (chunks.length === 0 || !question.trim()) return unjudged()
+
+    try {
+      const thinkingCapable = await this.ollamaService.checkModelHasThinking(model)
+      const response = await this.ollamaService.chat({
+        model,
+        messages: [
+          { role: 'system', content: SYSTEM_PROMPTS.relevance_check },
+          {
+            role: 'user',
+            content: buildJudgeInput(
+              question,
+              chunks.map((c) => ({
+                text: c.text,
+                // Titles only from content metadata. A filename is not a claim
+                // about what a passage says, and in the eval corpus it would
+                // hand the judge the answer ("distractor-…").
+                title:
+                  c.metadata?.full_title || c.metadata?.article_title || c.metadata?.archive_title,
+              }))
+            ),
+          },
+        ],
+        numPredict: RELEVANCE_CHECK_MAX_TOKENS,
+        temperature: 0,
+        think: false,
+        thinkingCapable,
+        format: ON_TOPIC_SCHEMA,
+      })
+
+      const verdict = resolveStructured(
+        response.message.content.trim(),
+        pickOnTopic,
+        response.structured === true
+      )
+      if (!verdict.ok) {
+        logger.warn(
+          `[RAG] Relevance check on "${model}" returned no usable verdict (${verdict.reason}); keeping all chunks`
+        )
+        return unjudged()
+      }
+
+      const ms = Date.now() - started
+      logger.debug(
+        `[RAG] Relevance check on "${model}": ${verdict.value ? 'on topic' : 'off topic'} (${chunks.length} chunk(s), ${ms}ms)`
+      )
+      return verdict.value
+        ? { kept: chunks, rejected: 0, judged: true, ms }
+        : { kept: [], rejected: chunks.length, judged: true, ms }
+    } catch (error) {
+      logger.warn(
+        `[RAG] Relevance check on "${model}" failed, keeping all chunks: ${error instanceof Error ? error.message : error}`
+      )
+      return unjudged()
+    }
+  }
+}

admin/app/utils/eval/retrieval_metrics.ts

@@ -225,6 +225,12 @@ export type RetrievalCaseResult = {
   reciprocalRank: number | null
   /** True when this case retrieved nothing at all. */
   empty: boolean
+  /**
+   * The highest-ranked chunk's scores, or null when nothing was retrieved. Shown
+   * for out-of-corpus leaks: the gap between the two says how much of a false
+   * positive the reranker's lexical boosts supplied.
+   */
+  top: { score: number; semanticScore: number | null } | null
 }
 
 export type RetrievalAggregate = {
@@ -275,6 +281,10 @@ export function scoreCase(c: RetrievalCase, kValues: number[] = DEFAULT_K_VALUES
     ndcg: byK((k) => ndcgAtK(c.retrieved, c.relevantDocIds, k)),
     reciprocalRank: reciprocalRank(c.retrieved, c.relevantDocIds),
     empty: c.retrieved.length === 0,
+    top:
+      c.retrieved.length === 0
+        ? null
+        : { score: c.retrieved[0].score, semanticScore: c.retrieved[0].semanticScore ?? null },
   }
 }
 

admin/app/utils/relevance_judge.ts

@@ -0,0 +1,107 @@
+/**
+ * The pure half of the retrieval relevance check (#1341): the grammar, the
+ * setting, and what the judge gets to read.
+ *
+ * Why a model and not another threshold: on a library holding general prose
+ * (Gutenberg, Wikipedia), the nearest chunk to *any* question scores as high as
+ * a genuinely relevant one. Measured on the eval corpus with its distractors,
+ * the weakest correct top hit scored 0.666 post-rerank while "write me a haiku
+ * about autumn leaves" pulled Walden at 0.705. No cutoff on that axis separates
+ * the two, so the decision has to be a judgement about the question.
+ *
+ * Why one verdict for the whole set rather than one per passage: asked
+ * passage-by-passage, every model tried (0.5B to 8B) rejected passages that
+ * were on-topic but did not spell the answer out, and cost 9-17 points of
+ * recall@5. "Are these about what was asked?" is the easier question, and on an
+ * 8B model it took off-topic leaks from 67% to 6% for ~5 points of recall. On a
+ * 3B model it rejects most real questions, which is why the check is opt-in.
+ *
+ * Pure so it unit-tests under bare node; RelevanceJudgeService owns the call.
+ */
+
+/**
+ * Longest excerpt of each passage the judge reads. At 700 characters the window
+ * routinely stopped a sentence short of the answer (wound packing, counterpoise
+ * length) and the judge rightly said the passage didn't cover it.
+ */
+export const JUDGE_EXCERPT_CHARS = 1500
+
+/** Anything shorter carries no topical signal; don't let it steer the excerpt. */
+const MIN_TERM_LENGTH = 4
+
+export type JudgeablePassage = { text: string; title?: string }
+
+export const ON_TOPIC_SCHEMA = {
+  type: 'object',
+  properties: { on_topic: { type: 'boolean' } },
+  required: ['on_topic'],
+  additionalProperties: false,
+} as const
+
+export function pickOnTopic(value: any): boolean | null {
+  return typeof value?.on_topic === 'boolean' ? value.on_topic : null
+}
+
+/**
+ * Interpret the stored `rag.relevanceCheck` setting. Off unless explicitly on:
+ * the check costs a model call per turn and hurts recall on small models, so
+ * nothing short of a deliberate choice should turn it on.
+ */
+export function isRelevanceCheckEnabled(raw: unknown): boolean {
+  return raw === true || raw === 'true'
+}
+
+/**
+ * The stretch of `text` most about the question, at most `maxChars` long.
+ *
+ * Chunks run ~3000 characters and the judge sees half of each, so which half
+ * matters. Picks the window holding the most occurrences of the question's
+ * longer terms, falling back to the start when none appear.
+ */
+export function excerptFor(text: string, question: string, maxChars = JUDGE_EXCERPT_CHARS): string {
+  const clean = text.replace(/\s+/g, ' ').trim()
+  if (clean.length <= maxChars) return clean
+
+  const terms = [
+    ...new Set(
+      question
+        .toLowerCase()
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter((t) => t.length >= MIN_TERM_LENGTH)
+    ),
+  ]
+  const lower = clean.toLowerCase()
+  const hits: number[] = []
+  for (const term of terms) {
+    for (let at = lower.indexOf(term); at !== -1; at = lower.indexOf(term, at + term.length)) {
+      hits.push(at)
+    }
+  }
+  if (hits.length === 0) return clean.slice(0, maxChars) + '…'
+  hits.sort((a, b) => a - b)
+
+  // Densest window: two pointers over the sorted hit positions.
+  let bestStart = hits[0]
+  let bestCount = 0
+  for (let lo = 0, hi = 0; hi < hits.length; hi++) {
+    while (hits[hi] - hits[lo] > maxChars) lo++
+    if (hi - lo + 1 > bestCount) {
+      bestCount = hi - lo + 1
+      bestStart = hits[lo]
+    }
+  }
+
+  // Open a little before the first hit so the sentence it sits in reads whole.
+  const start = Math.max(0, Math.min(bestStart - Math.floor(maxChars / 5), clean.length - maxChars))
+  const end = start + maxChars
+  return (start > 0 ? '…' : '') + clean.slice(start, end) + (end < clean.length ? '…' : '')
+}
+
+/** The user turn of the judge call: the question, then each passage. */
+export function buildJudgeInput(question: string, passages: JudgeablePassage[]): string {
+  const blocks = passages.map((p, i) => {
+    const heading = p.title ? `p${i + 1} (from "${p.title}")` : `p${i + 1}`
+    return `${heading}:\n${excerptFor(p.text, question)}`
+  })
+  return `Question: ${question.trim()}\n\n${blocks.join('\n\n')}`
+}

admin/commands/eval/retrieval.ts

@@ -33,6 +33,12 @@ export default class EvalRetrieval extends BaseCommand {
   })
   declare minFinalScore: string
 
+  @flags.string({
+    description:
+      'Run the retrieval relevance check on this model after the floor (default: off, keeping this tier model-free)',
+  })
+  declare judgeModel: string
+
   @flags.boolean({ description: 'Also score the raw dense, reranked, and diversified orderings' })
   declare ablate: boolean
 
@@ -55,6 +61,13 @@ export default class EvalRetrieval extends BaseCommand {
   @flags.boolean({ description: 'Write a JSON + Markdown report to tests/eval/reports/' })
   declare report: boolean
 
+  @flags.string({
+    description:
+      "Write every golden's full pre-floor candidate pool (boosted + cosine scores) to this JSON path, " +
+      'for simulating a relevance gate offline',
+  })
+  declare dump: string
+
   static options: CommandOptions = {
     startApp: true,
   }
@@ -93,6 +106,8 @@ export default class EvalRetrieval extends BaseCommand {
         scoreThreshold: this.threshold ? Number.parseFloat(this.threshold) : undefined,
         minFinalScore: this.minFinalScore ? Number.parseFloat(this.minFinalScore) : undefined,
         ablate: this.ablate,
+        dump: Boolean(this.dump),
+        judgeModel: this.judgeModel || undefined,
       })
       const elapsed = ((Date.now() - started) / 1000).toFixed(1)
 
@@ -100,6 +115,13 @@ export default class EvalRetrieval extends BaseCommand {
         `Params: topK=${result.params.topK} threshold=${result.params.scoreThreshold} ` +
           `minFinalScore=${result.params.minFinalScore}  (${elapsed}s)`
       )
+      if (result.judge) {
+        const j = result.judge
+        this.logger.info(
+          `Relevance check: ${j.model} · ${j.calls} calls · rejected ${j.rejected} chunk(s) · ` +
+            `${j.failures} failed open · mean ${Math.round(j.meanMs)}ms max ${Math.round(j.maxMs)}ms`
+        )
+      }
       this.logger.info('')
       this.printAggregate('OVERALL', result.overall, result.params.kValues)
 
@@ -133,8 +155,12 @@ export default class EvalRetrieval extends BaseCommand {
       this.logger.info('=== By tag ===')
       for (const [tag, agg] of Object.entries(result.byTag).sort()) {
         const k = result.params.kValues.includes(5) ? 5 : result.params.kValues[0]
+        // A refusal-only tag has no recall to report; what it has is the leak rate,
+        // which is the number #1341 is about.
+        const leak =
+          agg.nonEmptyRateOnRefusal === null ? '' : `  leaked=${pct(agg.nonEmptyRateOnRefusal)}`
         this.logger.info(
-          `  ${tag.padEnd(20)} n=${String(agg.cases).padStart(3)}  recall@${k}=${fmt(agg.recall[k])}  ndcg@${k}=${fmt(agg.ndcg[k])}`
+          `  ${tag.padEnd(20)} n=${String(agg.cases).padStart(3)}  recall@${k}=${fmt(agg.recall[k])}  ndcg@${k}=${fmt(agg.ndcg[k])}${leak}`
         )
       }
 
@@ -155,6 +181,32 @@ export default class EvalRetrieval extends BaseCommand {
         this.logger.info('Re-run with --verbose to see which questions and what they retrieved.')
       }
 
+      // The other failure: a question the corpus cannot answer that retrieved
+      // something anyway. Each of these becomes a citation the user sees.
+      const leaks = result.cases.filter((c) => c.expectRefusal && !c.empty)
+      if (leaks.length > 0) {
+        this.logger.info('')
+        this.logger.info(`${leaks.length} out-of-corpus question(s) retrieved something anyway`)
+        if (this.verbose) {
+          this.logger.info('')
+          this.logger.info('=== Leaks ===')
+          for (const leak of leaks) {
+            this.logger.info(`  ${leak.id}`)
+            const top = leak.top
+              ? `  (top final=${leak.top.score.toFixed(3)} semantic=${leak.top.semanticScore?.toFixed(3) ?? 'n/a'})`
+              : ''
+            this.logger.info(`    retrieved: ${leak.retrievedDocIds.join(', ')}${top}`)
+          }
+        }
+      }
+
+      if (this.dump && result.pools) {
+        const { writeFile } = await import('node:fs/promises')
+        await writeFile(this.dump, JSON.stringify({ fingerprint, params: result.params, pools: result.pools }, null, 2))
+        this.logger.info('')
+        this.logger.success(`Candidate pools written: ${this.dump}`)
+      }
+
       if (this.report) {
         const meta = await reportService.buildMeta('retrieval', fingerprint, {
           ...result.params,

admin/constants/kv_store.ts

@@ -18,6 +18,7 @@ export const SETTINGS_KEYS: KVStoreKey[] = [
     'rag.defaultIngestPolicy',
     'rag.enabled',
     'rag.minRelevance',
+    'rag.relevanceCheck',
     'autoUpdate.enabled',
     'autoUpdate.windowStart',
     'autoUpdate.windowEnd',

admin/constants/ollama.ts

@@ -149,6 +149,24 @@ export const RAG_DEFAULT_SCORE_THRESHOLD = 0.3
  * So nonEmptyRateOnRefusal bottoms out at 0.6 here, and a cutoff tuned to drive
  * it lower is cutting real documents.
  *
+ * What this floor cannot do (#1341). With general-prose distractors added to the
+ * corpus (fingerprint b4ebe5dce8699f5d: Gutenberg excerpts, the Python tutorial,
+ * MDN), the `off_topic` suite measures:
+ *
+ *     floor   core recall@5   off-topic questions that retrieve something
+ *     0.62        0.991                  67%   (22% without the distractors)
+ *     0.65        0.991                  33%
+ *     0.69        0.934                  11%
+ *
+ * The weakest correct top hit scores 0.666 while "write me a haiku about autumn
+ * leaves" pulls Walden at 0.705, so no value here separates the two, and a
+ * better-stocked library only makes it worse. 0.65 halves the leaks at no
+ * measured recall cost but sits 0.016 under that weakest hit, which is not the
+ * margin this default was chosen for. The coherent-but-off-topic case is left to
+ * the opt-in relevance check (app/utils/relevance_judge.ts). A cosine floor
+ * alongside this one was also tried and loses recall faster than it cuts leaks:
+ * the reranker's boosts lift relevant chunks more than irrelevant ones.
+ *
  * Unset `rag.minRelevance` resolves to this value; see app/utils/rag_relevance.ts.
  */
 export const RAG_MIN_FINAL_SCORE = 0.62
@@ -211,6 +229,13 @@ export const RAG_PLACEMENT: 'tail' | 'front' = 'tail'
  */
 export const QUERY_REWRITE_MAX_TOKENS = 160
 
+/**
+ * Token cap on the retrieval relevance check. The answer is `{"on_topic": true}`,
+ * under ten tokens; the rest is headroom, since a truncated verdict is discarded
+ * and the turn keeps its chunks unjudged.
+ */
+export const RELEVANCE_CHECK_MAX_TOKENS = 32
+
 /**
  * How long Ollama keeps a chat model — and its KV cache — resident after a
  * request.
@@ -344,6 +369,13 @@ Respond with JSON: {"suggestions": ["...", "...", "..."]}
   title_generation: `You are a title generator. Given the start of a conversation, generate a concise, descriptive title under 50 characters.
 
 Respond with JSON: {"title": "..."}`,
+  relevance_check: `You check whether search results are about what a user asked.
+
+Answer true if at least one passage is about the same specific subject as the question, even if it does not state the exact answer.
+
+Answer false if the passages only share a word or a loose theme with the question. Also answer false if the user is not asking for information at all: greetings, chit-chat, or asking you to produce something new such as a poem, a joke, a translation, or a letter.
+
+Respond with JSON: {"on_topic": true} or {"on_topic": false}`,
   query_rewrite: `
 You are a query rewriting assistant. Your task is to reformulate the user's latest question to include relevant context from the conversation history.
 

admin/inertia/pages/settings/models.tsx

@@ -28,7 +28,7 @@ export default function ModelsPage(props: {
   models: {
     availableModels: NomadOllamaModel[]
     installedModels: NomadInstalledModel[]
-    settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string; minRelevance: number; responseStyle: string }
+    settings: { chatSuggestionsEnabled: boolean; aiAssistantCustomName: string; remoteOllamaUrl: string; ollamaFlashAttention: boolean; autoThinking: boolean; tasksModel: string; ragEnabled: boolean; contextWindow: string; minRelevance: number; relevanceCheck: boolean; responseStyle: string }
     /** Effective window per installed model, as resolved by ContextWindowService. */
     resolvedContextWindows?: Record<string, number>
   }
@@ -108,6 +108,7 @@ export default function ModelsPage(props: {
   const [tasksModel, setTasksModel] = useState(props.models.settings.tasksModel)
   const [contextWindow, setContextWindow] = useState(props.models.settings.contextWindow)
   const [minRelevance, setMinRelevance] = useState(String(props.models.settings.minRelevance))
+  const [relevanceCheck, setRelevanceCheck] = useState(props.models.settings.relevanceCheck)
   const [responseStyle, setResponseStyle] = useState(props.models.settings.responseStyle)
   const [aiAssistantCustomName, setAiAssistantCustomName] = useState(
     props.models.settings.aiAssistantCustomName
@@ -441,6 +442,16 @@ export default function ModelsPage(props: {
                   updateSettingMutation.mutate({ key: 'rag.minRelevance', value: newVal })
                 }}
               />
+              <Switch
+                checked={relevanceCheck}
+                disabled={!ragEnabled}
+                onChange={(newVal) => {
+                  setRelevanceCheck(newVal)
+                  updateSettingMutation.mutate({ key: 'rag.relevanceCheck', value: newVal })
+                }}
+                label="Double-check knowledge base matches"
+                description="Before answering, ask the Tasks Model whether the passages found are actually about your question, and leave them out (and uncited) when they are not. Helps most with large libraries like Wikipedia or Project Gutenberg, where something always looks similar. Needs a capable Tasks Model (about 8B parameters or larger); smaller models reject too many good matches. Adds a short delay to every reply."
+              />
               <Select
                 name="responseStyle"
                 label="Response Style"

admin/tests/eval/README.md

@@ -39,10 +39,32 @@ invalidate every committed baseline.
 |---|---|
 | `core` | The default 99 questions. Retrieval and generation quality. |
 | `long_context` | Conversations long enough to exercise the budget planner. |
+| `off_topic` | Coherent questions the corpus cannot answer, plus distractor controls (#1341). |
 
 `--suite a,b` runs several. `eval:corpus` always validates every suite, since a
 malformed fixture should fail validation whether or not it is in the default run.
 
+### The `off_topic` suite
+
+Eighteen questions no document answers (general trivia, chit-chat, requests to
+write something, personal asks, and domains the corpus lacks, such as gardening)
+and seven controls that the `distractor-*` documents *do* answer.
+
+Every refusal case must retrieve **nothing**. `eval:retrieval` prints a `leaked=`
+rate per tag and, with `--verbose`, each leaked question with its top chunk's
+boosted and cosine scores. The controls keep a fix honest: a gate that stops the
+leaks by also rejecting "how do list comprehensions work in Python?" has traded
+one bug for another.
+
+The suite exists because the relevance floor looked fine on the original 28
+documents (22% of these leaked) and fails on a well-stocked library (67% leaked
+once Gutenberg prose and programming docs were added). A small corpus
+understates this failure, so keep the distractors in.
+
+Retrieval-only for now. The generation tier scores `expectRefusal` as "the model
+declined", and "Paris" is the right answer to the capital question as long as
+nothing is cited for it.
+
 ### The `long_context` suite
 
 Four conversations testing the two opposite ways context budgeting can fail:
@@ -159,8 +181,16 @@ node ace eval:retrieval --min-final-score=0.7 # sweep the post-rerank relevance
 node ace eval:retrieval --tag=multi-hop
 node ace eval:retrieval --verbose             # show every miss and what it retrieved
 node ace eval:retrieval --report              # write JSON + Markdown to reports/
+node ace eval:retrieval --suite=off_topic --judge-model=llama3:8b   # measure the relevance check
+node ace eval:retrieval --suite=core,off_topic --dump=pools.json    # candidate pools for offline gate sims
 ```
 
+`--judge-model` runs the opt-in relevance check (`rag.relevanceCheck`) after the
+floor and prints its latency and how often it failed open. It makes the run
+model-dependent, so it is never part of a baseline. `--dump` writes every
+golden's full pre-floor candidate pool with both scores, which lets you simulate
+a new gate in a few lines of script instead of a sweep per setting.
+
 **There are two cutoffs, on two different scores, and they do not interchange.**
 `--threshold` is what Qdrant applies to the raw cosine score, before reranking;
 `--min-final-score` (`RAG_MIN_FINAL_SCORE`) is the relevance floor applied to
@@ -237,7 +267,12 @@ manufacture a regression. Re-baseline instead.
 
 ## The corpus and the goldens
 
-- `corpus/*.md` — 28 short documents across NOMAD's real domains.
+- `corpus/*.md` — 28 short documents across NOMAD's real domains, plus six
+  `distractor-*` documents (Project Gutenberg excerpts, the Python tutorial, MDN
+  pages). The distractors stand in for a library with general prose in it, where
+  something is always semantically near the question. Sources and licences are
+  in each file's header: Gutenberg texts are public domain, the Python tutorial
+  is PSF License v2, MDN is CC-BY-SA 2.5.
 - `goldens/*.jsonl` — 99 questions, one JSON object per line.
 
 The corpus is built with deliberate traps, not just easy questions:
@@ -383,8 +418,8 @@ Read these before trusting a number.
 
 ```
 tests/eval/
-  corpus/         28 markdown fixtures — the frozen knowledge base
-  goldens/        99 questions as JSONL
+  corpus/         34 markdown fixtures (28 domain + 6 distractor) — the frozen knowledge base
+  goldens/        one JSONL file per suite (core: 99 questions)
   baselines/      <corpus-fingerprint>/*.json — COMMITTED; the gate compares against these
   reports/        run artifacts — gitignored
 ```

admin/tests/eval/corpus/distractor-mdn-css-flexbox.md

@@ -0,0 +1,438 @@
+# MDN: Basic concepts of flexbox
+
+Distractor document for the #1341 off-topic eval suite. From MDN Web Docs (github.com/mdn/content), by Mozilla Contributors, CC-BY-SA 2.5.
+
+The [flexible box layout](/en-US/docs/Web/CSS/Guides/Flexible_box_layout) module (usually referred to as flexbox) is a one-dimensional layout model for distributing space between items and includes numerous alignment capabilities. This article gives an outline of the main features of flexbox, which we will explore in more detail in the rest of these guides.
+
+When we describe flexbox as being one-dimensional we are describing the fact that flexbox deals with layout in one dimension at a time — either as a row or as a column. This can be contrasted with the two-dimensional model of [CSS Grid Layout](/en-US/docs/Web/CSS/Guides/Grid_layout), which controls columns and rows together.
+
+## The two axes of flexbox
+
+When working with flexbox you need to think in terms of two axes — the _main axis_ and the _cross axis_. The [main axis](#the_main_axis) is defined by the {{cssxref("flex-direction")}} property, and the [cross axis](#the_cross_axis) runs perpendicular to it. Everything we do with flexbox refers back to these axes, so it is worth understanding how they work from the outset.
+
+### The main axis
+
+The {{glossary("main axis")}} is defined by `flex-direction`, which has four possible values:
+
+- `row`
+- `row-reverse`
+- `column`
+- `column-reverse`
+
+Should you choose `row` or `row-reverse`, your main axis will run along the row in the **inline direction**.
+
+![If flex-direction is set to row the main axis runs along the row in the inline direction.](basics1.svg)
+
+Choose `column` or `column-reverse` and your main axis will run in the **block direction**, from the top of the page to the bottom.
+
+![If flex-direction is set to column the main axis runs in the block direction.](basics2.svg)
+
+### The cross axis
+
+The {{glossary("cross axis")}} runs perpendicular to the main axis. Therefore, if your `flex-direction` (main axis) is set to `row` or `row-reverse` the cross axis runs down the columns.
+
+![If flex-direction is set to row then the cross axis runs in the block direction.](basics3.svg)
+
+If your main axis is `column` or `column-reverse` then the cross axis runs along the rows.
+
+![If flex-direction is set to column then the cross axis runs in the inline direction.](basics4.svg)
+
+## Start and end lines
+
+Another vital area of understanding is how flexbox makes no assumption about the writing mode of the document. Flexbox doesn't just assume that all lines of text start at the top left of a document and run towards the right-hand side, with new lines appearing one under the other. Rather, it supports all writing modes, like other [logical properties and values](/en-US/docs/Web/CSS/Guides/Logical_properties_and_values).
+
+You can [read more about the relationship between flexbox and writing modes](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Relationship_with_other_layout_methods#writing_modes) in a later article; however, the following description should help explain why we do not talk about left and right and top and bottom when we describe the direction that our flex items flow in.
+
+If the `flex-direction` is `row` and I am working in English, then the start edge of the main axis will be on the left, the end edge on the right.
+
+![Working in English the start edge is on the left.](basics5.svg)
+
+If I were to work in Arabic, then the start edge of my main axis would be on the right and the end edge on the left.
+
+![The start edge in a RTL language is on the right.](basics6.svg)
+
+In both cases the start edge of the cross-axis is at the top of the flex container and the end edge at the bottom, as both languages have a horizontal writing mode.
+
+After a while, thinking about start and end rather than left and right becomes natural, and will be useful to you when dealing with other layout methods such as CSS Grid Layout which follow the same patterns.
+
+## The flex container
+
+An area of a document that is laid out using flexbox is called a **flex container**. To create a {{glossary("flex container")}}, set the area's {{cssxref("display")}} property to `flex`. When we do this, the direct children of that container become **flex items**. You can explicitly control whether the container itself is displayed inline or in block formatting context using `inline flex` or `inline-flex` for inline flex containers or `block flex` or `flex` for block level flex containers.
+
+### Initial values
+
+As with all properties in CSS, some initial values are defined, so the contents of a new flex container will behave in the following way:
+
+- Items display in a row (the {{cssxref("flex-direction")}} property's default value is `row`).
+- The items start from the start edge of the main axis.
+- The items do not stretch on the main dimension but can shrink (a flex-item's {{cssxref("flex-grow")}} property's default value is `0` and its {{cssxref("flex-shrink")}} property's default value is `1`).
+- The items will stretch to fill the size of the cross-axis (the {{cssxref("align-items")}} property's default value is `stretch`).
+- The flex-item's {{cssxref("flex-basis")}} property's default value is `auto`. This means that, in each case, it will be equal to the flex item {{cssxref("width")}} in horizontal writing mode, and the flex item {{cssxref("height")}} in vertical writing mode. If the corresponding `width`/`height` is also set to `auto`, the `flex-basis` `content` value is used instead.
+- All the items will be in a single row (the {{cssxref("flex-wrap")}} property's default value is `nowrap`), overflowing their container if their combined `width`/`height` exceeds the containing element `width`/`height`.
+
+The result of this is that your items will all line up in a row, using the size of the content as their size in the main axis. If there are more items than can fit in the container, they will not wrap but will instead overflow. If some items are taller than others, all items will stretch along the full length of the cross-axis.
+
+You can see in the live sample below how this looks. Click "Play" to open the example in the MDN Playground and edit the items or add new items to try out the initial behavior of flexbox:
+
+```html live-sample___the-flex-container
+<div class="box">
+  <div>One</div>
+  <div>Two</div>
+  <div>Three <br />has <br />extra <br />text</div>
+</div>
+```
+
+```css live-sample___the-flex-container
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+}
+
+.box {
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+}
+```
+
+{{EmbedLiveSample("the-flex-container")}}
+
+### Changing flex-direction
+
+Adding the {{cssxref("flex-direction")}} property to the flex container allows us to change the direction in which our flex items display. Setting `flex-direction: row-reverse` will keep the items displaying along the row, however the start and end lines are switched.
+
+If we change `flex-direction` to `column` the main axis switches and our items now display in a column. Set `column-reverse` and the start and end lines are again switched.
+
+The live sample below has `flex-direction` set to `row-reverse`. Try the other values — `row`, `column` and `column-reverse` — to see what happens to the content.
+
+```html live-sample___flex-direction
+<div class="box">
+  <div>One</div>
+  <div>Two</div>
+  <div>Three</div>
+</div>
+```
+
+```css live-sample___flex-direction
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+}
+
+.box {
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+  flex-direction: row-reverse;
+}
+```
+
+{{EmbedLiveSample("flex-direction")}}
+
+## Multi-line flex containers with flex-wrap
+
+While flexbox is a one dimensional model, it is possible to make flex items wrap across multiple lines. If you do this, you should consider each line as a new flex container. Any space distribution will happen across each line, without reference to the previous or subsequent lines.
+
+To cause wrapping behavior add the property {{cssxref("flex-wrap")}} with a value of `wrap`. Now, if your items are too large to all display in one line, they will wrap onto another line. The live sample below contains items that have been given a `width`. The total width of the items is too wide for the flex container. As `flex-wrap` is set to `wrap`, the items wrap across multiple lines. If you set it to `nowrap`, which is the initial value, they will shrink to fit the container. They shrink because they are using initial flexbox values, including `flex-shrink: 1`, that allows items to shrink. Using `nowrap` would cause an [overflow](/en-US/docs/Learn_web_development/Core/Styling_basics/Overflow) if the items were not able to shrink, or could not shrink small enough to fit.
+
+```html live-sample___flex-wrap
+<div class="box">
+  <div>One</div>
+  <div>Two</div>
+  <div>Three</div>
+</div>
+```
+
+```css live-sample___flex-wrap
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+  width: 200px;
+}
+
+.box {
+  width: 500px;
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+  flex-wrap: wrap;
+}
+```
+
+{{EmbedLiveSample("flex-wrap")}}
+
+Find out more about wrapping flex items in the guide [Mastering wrapping of flex items](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Wrapping_items).
+
+## The flex-flow shorthand
+
+You can combine the two properties `flex-direction` and `flex-wrap` into the {{cssxref("flex-flow")}} shorthand.
+
+In the live sample below, try changing the first value to one of the allowable values for `flex-direction` - `row`, `row-reverse`, `column` or `column-reverse`, and also change the second to `wrap` and `nowrap`.
+
+```html live-sample___flex-flow
+<div class="box">
+  <div>One</div>
+  <div>Two</div>
+  <div>Three</div>
+</div>
+```
+
+```css live-sample___flex-flow
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+  width: 200px;
+}
+
+.box {
+  width: 500px;
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+  flex-flow: row wrap;
+}
+```
+
+{{EmbedLiveSample("flex-flow")}}
+
+## Properties applied to flex items
+
+To control the inline-size of each flex item, we target them directly via three properties:
+
+- {{cssxref("flex-grow")}}
+- {{cssxref("flex-shrink")}}
+- {{cssxref("flex-basis")}}
+
+We will take a brief look at these properties below, but if you want more comprehensive information, take a look at the [Controlling ratios of flex items on the main axis](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Controlling_flex_item_ratios) guide.
+
+Before we can make sense of these properties we need to consider the concept of **available space**. What we are doing when we change the value of these flex properties is to change the way that available space is distributed among our items. This concept of available space is also important when we come to look at aligning items.
+
+If we have three 100 pixel-wide items in a container which is 500 pixels wide, then the space we need to lay out our items is 300 pixels. This leaves 200 pixels of available space. If we don't change the initial values then flexbox will put that space after the last item.
+
+![This flex container has available space after laying out the items.](basics7.svg)
+
+If we instead would like the items to grow and fill the space, then we need to have a method of distributing the leftover space between the items. The `flex` properties that we apply to the items themselves, enable dictating how that available space should be distributed among the sibling flex items.
+
+### The flex-basis property
+
+The `flex-basis` is what defines the size of that item in terms of the space it leaves as available space. The initial value of this property is `auto` — in this case the browser looks to see if the item has a size. In the example above, all of the items have a width of 100 pixels. This is used as the `flex-basis`.
+
+If the items don't have a size then the content's size is used as the flex-basis. This is why when we just declare `display: flex` on the parent to create flex items, the items all move into a row and take only as much space as they need to display their contents.
+
+### The flex-grow property
+
+With the `flex-grow` property set to a positive integer, if there is available space, the flex item can grow along the main axis from its `flex-basis`. Whether the item stretches to take up all the available space on that axis, or just a portion of the available space depends on if the other items are allowed to grow too and the value of their `flex-grow` properties.
+
+Each item with a positive value consumes a portion of any available space based on their `flex-grow` value. If we gave all of our items in the example above a `flex-grow` value of 1 then the available space in the flex container would be equally shared between our items and they would stretch to fill the container on the main axis. If we give our first item a `flex-grow` value of 2, and the other items a value of 1 each, there are a total of 4 parts; 2 parts of the available space will be given to the first item (100px out of 200px in the case of the example above) and 1 part each the other two (50px each out of the 200px total).
+
+### The flex-shrink property
+
+Where the `flex-grow` property deals with adding space in the main axis, the `flex-shrink` property controls how it is taken away. If we do not have enough space in the container to lay out our items, and `flex-shrink` is set to a positive integer, then the item can become smaller than the `flex-basis`. As with `flex-grow`, different values can be assigned in order to cause one item to shrink faster than others — an item with a higher value set for `flex-shrink` will shrink faster than its siblings that have lower values.
+
+An item can shrink down to its {{cssxref("min-content")}} size. This minimum size is taken into account while working out the actual amount of shrinkage that will happen, which means that `flex-shrink` has the potential to appear less consistent than `flex-grow` in behavior. We'll therefore take a more detailed look at how this algorithm works in the article [Controlling ratios of items along the main axis](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Controlling_flex_item_ratios).
+
+> [!NOTE]
+> These values for `flex-grow` and `flex-shrink` are proportions. Typically if we had all of our items set to `flex: 1 1 200px` and then wanted one item to grow at twice the rate, we would set that item to `flex: 2 1 200px`. However you could also use `flex: 10 1 200px` and `flex: 20 1 200px` if you wanted.
+
+### Shorthand values for the flex properties
+
+You will very rarely see the `flex-grow`, `flex-shrink`, and `flex-basis` properties used individually; instead they are combined into the {{cssxref("flex")}} shorthand. The `flex` shorthand allows you to set the three values in this order — `flex-grow`, `flex-shrink`, `flex-basis`.
+
+The live sample below allows you to test out the different values of the flex shorthand; remember that the first value is `flex-grow`. Giving this a positive value means the item can grow. The second is `flex-shrink` — with a positive value the items can shrink, but only if their total values overflow the main axis. The final value is `flex-basis`; this is the value the items are using as their base value to grow and shrink from.
+
+```html live-sample___flex-properties
+<div class="box">
+  <div class="one">One</div>
+  <div class="two">Two</div>
+  <div class="three">Three</div>
+</div>
+```
+
+```css live-sample___flex-properties
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+}
+
+.box {
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+}
+
+.one {
+  flex: 1 1 auto;
+}
+
+.two {
+  flex: 1 1 auto;
+}
+
+.three {
+  flex: 1 1 auto;
+}
+```
+
+{{EmbedLiveSample("flex-properties")}}
+
+There are also some predefined shorthand values which cover most of the use cases. You will often see these used in tutorials, and in many cases these are all you will need to use. The predefined values are as follows:
+
+- `flex: initial`
+- `flex: auto`
+- `flex: none`
+- `flex: <positive-number>`
+
+The `initial` value is a [CSS-wide keyword](/en-US/docs/Web/CSS/Reference/Values/Data_types#css-wide_keywords) that represents the initial value for a property. Setting `flex: initial` resets the item to the [initial values](#initial_values) of the three longhand properties, which is the same as `flex: 0 1 auto`. The initial value of `flex-grow` is `0`, so items will not grow larger than their `flex-basis` size. The initial value of `flex-shrink` is `1`, so items can shrink if they need to rather than overflowing. The initial value of `flex-basis` is `auto`. Items will either use any size set on the item in the main dimension, or they will get their size from the content size.
+
+Using `flex: auto` is the same as using `flex: 1 1 auto`; this is similar to `flex: initial`, except that the items can grow and fill the container as well as shrink if needed.
+
+Using `flex: none` will create fully inflexible flex items. It is as if you wrote `flex: 0 0 auto`. The items cannot grow or shrink and will be laid out using flexbox with a `flex-basis` of `auto`.
+
+The shorthand you often see in tutorials is `flex: 1` or `flex: 2` and so on. This is the same as writing `flex: 1 1 0` or `flex: 2 1 0` and so on, respectively. The items get minimum size due to `flex-basis: 0` and then proportionally grow to fill the available space. In this case, the `flex-shrink` value of `1` is redundant because the items start with minimum size — they're not given any size that could cause them to overflow the flex container.
+
+Try these shorthand values in the live sample below.
+
+```html live-sample___flex-shorthands
+<div class="box">
+  <div class="one">One</div>
+  <div class="two">Two</div>
+  <div class="three">Three</div>
+</div>
+```
+
+```css live-sample___flex-shorthands
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+}
+
+.box {
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+}
+
+.one {
+  flex: 1;
+}
+
+.two {
+  flex: 1;
+}
+
+.three {
+  flex: 1;
+}
+```
+
+{{EmbedLiveSample("flex-shorthands")}}
+
+## Alignment, justification and distribution of free space between items
+
+A key feature of flexbox is the ability to align and justify items on the main- and cross-axes, and to distribute space between flex items. Note that these properties are set on the flex container, not on the items themselves.
+
+### align-items
+
+The {{cssxref("align-items")}} property aligns all the flex items on the cross axis.
+
+The initial value for this property is `stretch` and is why flex items stretch to the height of the flex container by default (or the width if `flex-direction` is set to `column` or `column-reverse`). This height may come from the tallest item in the container or the size set on the flex container itself.
+
+You could instead set `align-items` to `flex-start`, or simply `start`, in order to make the items line up at the start of the flex container, `flex-end`, or just `end`, to align them to the end, or `center` to align them in the center. Try this in the live sample — I have given the flex container a height in order that you can see how the items can be moved around inside the container. See what happens if you set the value of align-items to:
+
+- `stretch`
+- `flex-start`
+- `flex-end`
+- `start`
+- `end`
+- `center`
+- `baseline`
+- `last baseline`
+
+```html live-sample___align-items
+<div class="box">
+  <div>One</div>
+  <div>Two</div>
+  <div>Three <br />has <br />extra <br />text</div>
+</div>
+```
+
+```css live-sample___align-items
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+}
+
+.box {
+  width: 500px;
+  height: 130px;
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+  align-items: flex-start;
+}
+```
+
+{{EmbedLiveSample("align-items")}}
+
+The `align-items` is set on the flex container and impacts all the flex items. If you want to align a flex item differently from others, you can set the {{cssxref("align-self")}} on the flex item.
+
+### justify-content
+
+The {{cssxref("justify-content")}} property is used to align the items on the main axis, the direction in which `flex-direction` has set the flow. The initial value is `flex-start` which will line the items up at the start edge of the container, but you could also set the value to `flex-end` to line them up at the end, or `center` to line them up in the center.
+
+You can also use the value `space-between` to take all the spare space after the items have been laid out, and share it out evenly between the items so there will be an equal amount of space between each item. To cause an equal amount of space on the right and left (or top and bottom for columns) of each item use the value `space-around`. With `space-around`, items have a half-size space on either end. Or, to cause items to have equal space around them use the value `space-evenly`. With `space-evenly`, items have a full-size space on either end.
+
+Try the following values of `justify-content` in the live sample:
+
+- `start`
+- `end`
+- `left`
+- `right`
+- `normal`
+- `flex-start`
+- `flex-end`
+- `center`
+- `space-around`
+- `space-between`
+- `space-evenly`
+- `stretch`
+
+```html live-sample___justify-content
+<div class="box">
+  <div>One</div>
+  <div>Two</div>
+  <div>Three</div>
+</div>
+```
+
+```css live-sample___justify-content
+.box > * {
+  border: 2px solid rgb(96 139 168);
+  border-radius: 5px;
+  background-color: rgb(96 139 168 / 0.2);
+}
+
+.box {
+  border: 2px dotted rgb(96 139 168);
+  display: flex;
+  justify-content: flex-start;
+}
+```
+
+{{EmbedLiveSample("justify-content")}}
+
+The article [Aligning items in a flex container](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Aligning_items) explores these properties in more depth, in order to have a better understanding of how they work. These basic examples, however, are useful in the majority of use cases.
+
+### justify-items
+
+The {{cssxref("justify-items")}} property is ignored in flexbox layouts.
+
+### place-items and place-content
+
+The {{cssxref("place-items")}} property is a shorthand property for `align-items` and `justify-items`. If set on a flex container, it will set the alignment but not the justification, as `justify-items` is ignored in flexbox.
+
+There is another shorthand property, {{cssxref("place-content")}}, that defines the {{cssxref("align-content")}} and `justify-content` properties. The `align-content` property only affects flex containers that wrap, and is discussed in [Aligning items in a flex container](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Aligning_items).
+
+## Next steps
+
+After reading this article you should have an understanding of the basic features of flexbox. In the next article, we will look at [how this specification relates to other parts of CSS](/en-US/docs/Web/CSS/Guides/Flexible_box_layout/Relationship_with_other_layout_methods).

admin/tests/eval/goldens/off_topic.jsonl

@@ -0,0 +1,32 @@
+// #1341: coherent questions the corpus cannot answer. Every refusal case here must retrieve NOTHING.
+// The distractor-* documents (Gutenberg prose, Python tutorial, MDN) sit near everything in embedding
+// space, which is what makes the relevance floor fire on a well-stocked library and not on a small one.
+// Retrieval-only for now: generation scores expectRefusal as "the model declined", and "Paris" is the
+// right answer to the capital question as long as nothing is cited for it.
+{"id": "offtopic-issue-gibberish-01", "query": "zxqw plorbnat frunkle wibbet 7734 quozz", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "gibberish", "issue-1341"]}
+{"id": "offtopic-issue-banana-01", "query": "Reply with exactly the single word: banana", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "instruction", "issue-1341"]}
+{"id": "offtopic-issue-france-01", "query": "What is the capital city of France?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "general-knowledge", "issue-1341"]}
+{"id": "offtopic-issue-birthday-01", "query": "My sister's birthday is next Tuesday. What should I get her?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "personal", "issue-1341"]}
+{"id": "offtopic-issue-raised-bed-01", "query": "How deep should soil be in a raised garden bed?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "missing-domain", "issue-1341"]}
+{"id": "offtopic-hamlet-01", "query": "Who wrote Hamlet?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "general-knowledge"]}
+{"id": "offtopic-planets-01", "query": "How many planets are in the solar system?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "general-knowledge"]}
+{"id": "offtopic-celsius-01", "query": "What is 30 degrees Celsius in Fahrenheit?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "general-knowledge"]}
+{"id": "offtopic-haiku-01", "query": "Write me a haiku about autumn leaves.", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "instruction"]}
+{"id": "offtopic-joke-01", "query": "Tell me a joke.", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "instruction"]}
+{"id": "offtopic-greeting-01", "query": "Hi there, how are you doing today?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "instruction"]}
+{"id": "offtopic-translate-01", "query": "Translate 'good morning' into Spanish.", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "instruction"]}
+{"id": "offtopic-puppy-name-01", "query": "What should I name my new golden retriever puppy?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "personal"]}
+{"id": "offtopic-resume-01", "query": "Can you help me write a cover letter for a marketing job?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "personal"]}
+{"id": "offtopic-tomatoes-01", "query": "When should I transplant tomato seedlings outdoors?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "missing-domain"]}
+{"id": "offtopic-oil-change-01", "query": "How often should I change the oil in my car?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "missing-domain"]}
+{"id": "offtopic-sourdough-01", "query": "How do I feed a sourdough starter?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "missing-domain"]}
+{"id": "offtopic-mortgage-01", "query": "Should I refinance my mortgage if rates drop by half a percent?", "turns": [], "relevantDocIds": [], "mustInclude": [], "mustNotInclude": [], "expectRefusal": true, "tags": ["off-topic", "missing-domain"]}
+// Controls: questions the distractors DO answer. A gate that fixes the cases above by rejecting these
+// has traded one bug for another, so these must keep retrieving their document.
+{"id": "distractor-python-listcomp-01", "query": "How do list comprehensions work in Python?", "turns": [], "relevantDocIds": ["distractor-python-tutorial"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
+{"id": "distractor-python-exceptions-01", "query": "How do I catch an exception and still run cleanup code in Python?", "turns": [], "relevantDocIds": ["distractor-python-tutorial"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
+{"id": "distractor-js-promises-01", "query": "How do I chain promises in JavaScript and handle errors?", "turns": [], "relevantDocIds": ["distractor-mdn-javascript-guide"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
+{"id": "distractor-css-flex-01", "query": "How do I center items along the main axis of a flex container?", "turns": [], "relevantDocIds": ["distractor-mdn-css-flexbox"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
+{"id": "distractor-walden-beans-01", "query": "What did Thoreau write about hoeing his bean field at Walden?", "turns": [], "relevantDocIds": ["distractor-gutenberg-walden"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
+{"id": "distractor-holmes-redheaded-01", "query": "What was the Red-Headed League in the Sherlock Holmes story?", "turns": [], "relevantDocIds": ["distractor-gutenberg-sherlock-holmes"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}
+{"id": "distractor-pride-darcy-01", "query": "What did Mr. Darcy say about Elizabeth Bennet at the ball when he refused to dance with her?", "turns": [], "relevantDocIds": ["distractor-gutenberg-pride-and-prejudice"], "mustInclude": [], "mustNotInclude": [], "expectRefusal": false, "tags": ["distractor-control"]}

admin/tests/unit/rag_retrieval_toggle.spec.ts

@@ -22,8 +22,18 @@ import type { OllamaChatMessage } from '../../types/ollama.js'
 import type { RetrievedChunk } from '../../types/rag.js'
 
 /** Records every call so the tests can assert on what was *not* run. */
-function makeFakes(opts: { nomadMd?: string | null; searchResults?: RetrievedChunk[] } = {}) {
-  const calls = { hasDocuments: 0, search: 0, chat: 0 }
+function makeFakes(
+  opts: {
+    nomadMd?: string | null
+    searchResults?: RetrievedChunk[]
+    /** What the `rag.relevanceCheck` setting resolves to. Null (off) by default. */
+    checkModel?: string | null
+    /** The check's verdict when it runs. */
+    onTopic?: boolean
+  } = {}
+) {
+  const calls = { hasDocuments: 0, search: 0, chat: 0, judge: 0 }
+  const judged = { question: '', model: '' }
   const args = { minFinalScore: undefined as number | undefined }
 
   const ragService = {
@@ -65,6 +75,9 @@ function makeFakes(opts: { nomadMd?: string | null; searchResults?: RetrievedChu
     async getModels() {
       return []
     },
+    async checkModelHasThinking() {
+      return false
+    },
   }
 
   const nomadMdService = {
@@ -87,15 +100,35 @@ function makeFakes(opts: { nomadMd?: string | null; searchResults?: RetrievedChu
     },
   }
 
+  // Off unless a test turns it on, matching the setting's default.
+  const relevanceJudge = {
+    async resolveModel() {
+      return opts.checkModel ?? null
+    },
+    async judge(question: string, chunks: RetrievedChunk[], model: string) {
+      calls.judge++
+      judged.question = question
+      judged.model = model
+      const onTopic = opts.onTopic ?? true
+      return {
+        kept: onTopic ? chunks : [],
+        rejected: onTopic ? 0 : chunks.length,
+        judged: true,
+        ms: 7,
+      }
+    },
+  }
+
   const service = new RagPipelineService(
     ollamaService as any,
     ragService as any,
     nomadMdService as any,
     contextWindowService as any,
-    tokenCalibration as any
+    tokenCalibration as any,
+    relevanceJudge as any
   )
 
-  return { service, calls, args }
+  return { service, calls, args, judged }
 }
 
 const userTurn: OllamaChatMessage[] = [{ role: 'user', content: 'how do I purify water?' }]
@@ -206,3 +239,75 @@ test.group('buildPrompt | relevance floor', () => {
     assert.equal(args.minFinalScore, 0.9)
   })
 })
+
+test.group('buildPrompt | relevance check', () => {
+  test('does not run when the setting is off', async ({ assert }) => {
+    // Off is the default: the check is a model call per turn and only pays off
+    // on ~8B models, so it must cost nothing until someone opts in.
+    const { service, calls } = makeFakes()
+
+    const trace = await service.buildPrompt(userTurn, 'llama3.1:8b', {})
+
+    assert.equal(calls.judge, 0)
+    assert.isNull(trace.relevanceCheckModel)
+    assert.lengthOf(trace.injected, 1)
+  })
+
+  test('an off-topic verdict leaves nothing to inject or cite', async ({ assert }) => {
+    // #1341: the floor kept a chunk, the check says it is not about the
+    // question, so the prompt gets no context block and the answer no sources.
+    const { service, calls } = makeFakes({ checkModel: 'llama3:8b', onTopic: false })
+
+    const trace = await service.buildPrompt(userTurn, 'llama3.1:8b', {})
+
+    assert.equal(calls.judge, 1)
+    assert.deepEqual(trace.retrieved, [])
+    assert.deepEqual(trace.injected, [])
+    assert.equal(trace.chunksRejectedByCheck, 1)
+    assert.equal(trace.relevanceCheckModel, 'llama3:8b')
+    assert.isFalse(systemContents(trace.messages).some((c) => c.includes('[Context 1')))
+  })
+
+  test('an on-topic verdict keeps what the floor kept', async ({ assert }) => {
+    const { service } = makeFakes({ checkModel: 'llama3:8b', onTopic: true })
+
+    const trace = await service.buildPrompt(userTurn, 'llama3.1:8b', {})
+
+    assert.lengthOf(trace.injected, 1)
+    assert.equal(trace.chunksRejectedByCheck, 0)
+    assert.equal(trace.timings.relevanceCheckMs, 7)
+  })
+
+  test('judges the retrieval query, not the raw message', async ({ assert }) => {
+    // On a follow-up the rewrite is the standalone question; judging "what about
+    // the second one?" against the passages would reject everything.
+    const { service, judged } = makeFakes({ checkModel: 'llama3:8b' })
+    const conversation: OllamaChatMessage[] = [
+      { role: 'user', content: 'how do I purify water?' },
+      { role: 'assistant', content: 'Boil it or use chlorine.' },
+      { role: 'user', content: 'how long for the first one?' },
+    ]
+
+    await service.buildPrompt(conversation, 'llama3.1:8b', {})
+
+    assert.equal(judged.question, 'rewritten query')
+  })
+
+  test('an explicit null skips it whatever the setting says', async ({ assert }) => {
+    // The eval harness pins it off so one machine's setting cannot move scores.
+    const { service, calls } = makeFakes({ checkModel: 'llama3:8b', onTopic: false })
+
+    const trace = await service.buildPrompt(userTurn, 'llama3.1:8b', { relevanceCheckModel: null })
+
+    assert.equal(calls.judge, 0)
+    assert.lengthOf(trace.injected, 1)
+  })
+
+  test('skips the call when the floor already left nothing', async ({ assert }) => {
+    const { service, calls } = makeFakes({ checkModel: 'llama3:8b', searchResults: [] })
+
+    await service.buildPrompt(userTurn, 'llama3.1:8b', { minFinalScore: 0.62 })
+
+    assert.equal(calls.judge, 0)
+  })
+})

admin/tests/unit/relevance_judge.spec.ts

@@ -0,0 +1,74 @@
+/**
+ * The pure half of the retrieval relevance check (#1341).
+ *
+ * What these lock in: the setting stays off unless someone deliberately turned
+ * it on, a verdict is only ever read from a well-formed object, and the judge
+ * reads the part of a long chunk that is about the question rather than
+ * whatever happens to come first.
+ *
+ * Pure functions only — no MySQL, Redis, Qdrant, or Ollama needed:
+ *   npm run test:unit
+ */
+import * as assert from 'node:assert/strict'
+import { test } from 'node:test'
+
+import {
+  buildJudgeInput,
+  excerptFor,
+  isRelevanceCheckEnabled,
+  pickOnTopic,
+} from '../../app/utils/relevance_judge.js'
+
+test('relevance check: off unless explicitly on', () => {
+  assert.equal(isRelevanceCheckEnabled(true), true)
+  assert.equal(isRelevanceCheckEnabled('true'), true)
+  // Unset, cleared, or anything odd must not start costing a model call per turn.
+  for (const raw of [null, undefined, false, 'false', '', 'on', 'auto', 1]) {
+    assert.equal(isRelevanceCheckEnabled(raw), false, `expected ${JSON.stringify(raw)} to read as off`)
+  }
+})
+
+test('relevance check: a verdict is read only from a boolean on_topic', () => {
+  assert.equal(pickOnTopic({ on_topic: true }), true)
+  assert.equal(pickOnTopic({ on_topic: false }), false)
+  // A string "false" is truthy; treating it as a verdict would keep chunks the
+  // model rejected. Anything that isn't a real boolean is no verdict at all.
+  assert.equal(pickOnTopic({ on_topic: 'false' }), null)
+  assert.equal(pickOnTopic({}), null)
+  assert.equal(pickOnTopic(null), null)
+})
+
+test('excerpt: a short passage is passed through whole', () => {
+  assert.equal(excerptFor('Boil   water\nfor one minute.', 'boil water', 100), 'Boil water for one minute.')
+})
+
+test('excerpt: a long passage is cut to the window about the question', () => {
+  // The answer sits at the end of a long chunk. Taking the start would show the
+  // judge only filler, and it would rightly say the passage is off-topic.
+  const filler = 'Lorem ipsum dolor sit amet. '.repeat(40)
+  const text = `${filler}A counterpoise for a two meter handheld is a wire about 19 inches long.`
+  const excerpt = excerptFor(text, 'How long should a counterpoise be?', 200)
+
+  assert.match(excerpt, /counterpoise for a two meter handheld/)
+  assert.ok(excerpt.length <= 202, 'excerpt must stay within the window plus ellipses')
+  assert.ok(excerpt.startsWith('…'), 'a window that skips the start says so')
+})
+
+test('excerpt: with no term in the passage, falls back to the start', () => {
+  const text = 'x'.repeat(500)
+  assert.equal(excerptFor(text, 'unrelated question', 100), 'x'.repeat(100) + '…')
+})
+
+test('judge input: question first, then each passage under its key and title', () => {
+  const input = buildJudgeInput('  What is declination? ', [
+    { text: 'The angle between true and magnetic north.', title: 'Map and Compass' },
+    { text: 'Unrelated text.' },
+  ])
+
+  assert.equal(
+    input,
+    'Question: What is declination?\n\n' +
+      'p1 (from "Map and Compass"):\nThe angle between true and magnetic north.\n\n' +
+      'p2:\nUnrelated text.'
+  )
+})

admin/types/kv_store.ts

@@ -14,6 +14,10 @@ export const KV_STORE_SCHEMA = {
   // Stored as the number rather than a preset name so retuning the presets in
   // Settings > Models cannot invalidate a value someone already saved.
   'rag.minRelevance':           'string',
+  // Whether a model double-checks that retrieved chunks are about the question
+  // before they are injected and cited (#1341). Unset means OFF: the check costs
+  // a model call per turn and only pays off on ~8B-class models.
+  'rag.relevanceCheck':         'boolean',
   'system.updateAvailable':     'boolean',
   'system.latestVersion':       'string',
   'system.earlyAccess':         'boolean',

admin/types/rag.ts

@@ -48,7 +48,7 @@ export type RerankedRAGResult = Omit<RAGResult, 'keywords'> & {
 }
 
 /** One entry in a recorded retrieval stage: just enough to score a ranking. */
-export type StageEntry = { source?: string; score: number }
+export type StageEntry = { source?: string; score: number; semanticScore?: number }
 
 /**
  * The three ranked lists retrieval produces internally, captured so the eval
@@ -62,6 +62,13 @@ export type RetrievalStages = {
   dense?: StageEntry[]
   reranked?: StageEntry[]
   diversified?: StageEntry[]
+  /**
+   * The whole reranked candidate pool, unsliced and before the relevance floor,
+   * with both the boosted and the raw cosine score. What `eval:retrieval --dump`
+   * writes out, so a candidate gate can be simulated offline against every
+   * golden instead of costing a full sweep per setting.
+   */
+  candidates?: StageEntry[]
 }
 
 /**
@@ -117,6 +124,10 @@ export type PipelineOptions = {
    *  setting; the eval harness passes an explicit value so its numbers cannot
    *  depend on how one machine's slider happens to be set. */
   minFinalScore?: number
+  /** Model for the relevance check, or null to skip it. Unset resolves the
+   *  user's `rag.relevanceCheck` setting; the eval harness passes an explicit
+   *  value for the same reason it pins `minFinalScore`. */
+  relevanceCheckModel?: string | null
 }
 
 /**
@@ -139,7 +150,7 @@ export type PipelineTrace = {
   /** Generation cap, so the answer cannot run past the end of the window. */
   numPredict: number | undefined
   contextLimits: { maxResults: number; maxTokens: number }
-  timings: { rewriteMs: number; retrievalMs: number }
+  timings: { rewriteMs: number; retrievalMs: number; relevanceCheckMs: number }
   /**
    * The relevance floor this turn was retrieved under, and how many candidates
    * fell below it. `chunksBelowFloor > 0` with `retrieved.length === 0` is the
@@ -148,6 +159,10 @@ export type PipelineTrace = {
    */
   minFinalScore: number
   chunksBelowFloor: number
+  /** The model that ran the relevance check this turn, or null when it did not run. */
+  relevanceCheckModel: string | null
+  /** Chunks the check dropped after the floor had kept them. */
+  chunksRejectedByCheck: number
   /**
    * What the budget planner decided: how the window was spent and what was left
    * out. Undefined only when planning was bypassed. The eval harness reads this